{
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/peremartra/Large-Language-Model-Notebooks-Course/blob/main/4-Evaluating%20LLMs/evaluating_rag_giskard.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "<div align=\"center\">\n",
        "<h1><a href=\"https://github.com/peremartra/Large-Language-Model-Notebooks-Course\">Learn by Doing LLM Projects</a></h1>\n",
        "    <h3>Understand And Apply Large Language Models</h3>\n",
        "    <h2>Evaluating a RAG solution with Giskard</h2>\n",
        "    <p>by <b>Pere Martra</b></p>\n",
        "</div>\n",
        "\n",
        "<div align=\"center\">\n",
        "    &nbsp;\n",
        "    <a target=\"_blank\" href=\"https://www.linkedin.com/in/pere-martra/\"><img src=\"https://img.shields.io/badge/style--5eba00.svg?label=LinkedIn&logo=linkedin&style=social\"></a>\n",
        "    \n",
        "</div>\n",
        "\n"
      ],
      "metadata": {
        "id": "f3cvnSVemTSU"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "In the final part of this notebook, you will find the necessary code to evaluate the suitability of the responses provided by the Agent using Giskard."
      ],
      "metadata": {
        "id": "DUORXVgTmNYj"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "#Installing libraries & Loading Dataset"
      ],
      "metadata": {
        "id": "KNW-LniD2I2o"
      }
    },
    {
      "cell_type": "code",
      "execution_count": 85,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "pva9ehKXUpU2",
        "outputId": "71d14542-b9f9-4e19-eb4a-cb11142f8355"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\u001b[?25l     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m0.0/812.8 kB\u001b[0m \u001b[31m?\u001b[0m eta \u001b[36m-:--:--\u001b[0m\r\u001b[2K     \u001b[91m━━━━━━━━━━━━━━━\u001b[0m\u001b[91m╸\u001b[0m\u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m317.4/812.8 kB\u001b[0m \u001b[31m9.3 MB/s\u001b[0m eta \u001b[36m0:00:01\u001b[0m\r\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m812.8/812.8 kB\u001b[0m \u001b[31m14.6 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: langchainhub in /usr/local/lib/python3.10/dist-packages (0.1.15)\n",
            "Requirement already satisfied: requests<3,>=2 in /usr/local/lib/python3.10/dist-packages (from langchainhub) (2.31.0)\n",
            "Requirement already satisfied: types-requests<3.0.0.0,>=2.31.0.2 in /usr/local/lib/python3.10/dist-packages (from langchainhub) (2.31.0.20240406)\n",
            "Requirement already satisfied: charset-normalizer<4,>=2 in /usr/local/lib/python3.10/dist-packages (from requests<3,>=2->langchainhub) (3.3.2)\n",
            "Requirement already satisfied: idna<4,>=2.5 in /usr/local/lib/python3.10/dist-packages (from requests<3,>=2->langchainhub) (3.6)\n",
            "Requirement already satisfied: urllib3<3,>=1.21.1 in /usr/local/lib/python3.10/dist-packages (from requests<3,>=2->langchainhub) (2.0.7)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.10/dist-packages (from requests<3,>=2->langchainhub) (2024.2.2)\n"
          ]
        }
      ],
      "source": [
        "!pip install -q langchain==0.1.14\n",
        "!pip install -q langchain-openai==0.1.1\n",
        "!pip install langchainhub==0.1.15\n",
        "#!pip install --upgrade -q tiktoken==0.5.2\n",
        "!pip install -q datasets==2.16.1\n",
        "!pip install -q chromadb==0.4.22"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "!pip install -q openai --upgrade"
      ],
      "metadata": {
        "id": "ijC0kDQM8e-d"
      },
      "execution_count": 86,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "We will download the dataset from the Hugging Face datasets library. It's a dataset with information about diseases."
      ],
      "metadata": {
        "id": "KEwdHiGFkK_S"
      }
    },
    {
      "cell_type": "code",
      "execution_count": 87,
      "metadata": {
        "id": "laSDMjqQXuj-"
      },
      "outputs": [],
      "source": [
        "from datasets import load_dataset\n",
        "\n",
        "data = load_dataset(\"keivalya/MedQuad-MedicalQnADataset\", split='train')\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 88,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 484
        },
        "id": "JnWZTcJiXzor",
        "outputId": "aa9e33d6-99a0-491f-94b1-4df6ab4ec902"
      },
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "             qtype                                           Question  \\\n",
              "0   susceptibility  Who is at risk for Lymphocytic Choriomeningiti...   \n",
              "1         symptoms  What are the symptoms of Lymphocytic Choriomen...   \n",
              "2   susceptibility  Who is at risk for Lymphocytic Choriomeningiti...   \n",
              "3  exams and tests  How to diagnose Lymphocytic Choriomeningitis (...   \n",
              "4        treatment  What are the treatments for Lymphocytic Chorio...   \n",
              "5       prevention  How to prevent Lymphocytic Choriomeningitis (L...   \n",
              "6      information          What is (are) Parasites - Cysticercosis ?   \n",
              "7   susceptibility    Who is at risk for Parasites - Cysticercosis? ?   \n",
              "8  exams and tests        How to diagnose Parasites - Cysticercosis ?   \n",
              "9        treatment  What are the treatments for Parasites - Cystic...   \n",
              "\n",
              "                                              Answer  \n",
              "0  LCMV infections can occur after exposure to fr...  \n",
              "1  LCMV is most commonly recognized as causing ne...  \n",
              "2  Individuals of all ages who come into contact ...  \n",
              "3  During the first phase of the disease, the mos...  \n",
              "4  Aseptic meningitis, encephalitis, or meningoen...  \n",
              "5  LCMV infection can be prevented by avoiding co...  \n",
              "6  Cysticercosis is an infection caused by the la...  \n",
              "7  Cysticercosis is an infection caused by the la...  \n",
              "8  If you think that you may have cysticercosis, ...  \n",
              "9  Some people with cysticercosis do not need to ...  "
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-c60464df-e98f-42d2-8b01-1aafc8d5698d\" class=\"colab-df-container\">\n",
              "    <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>qtype</th>\n",
              "      <th>Question</th>\n",
              "      <th>Answer</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>susceptibility</td>\n",
              "      <td>Who is at risk for Lymphocytic Choriomeningiti...</td>\n",
              "      <td>LCMV infections can occur after exposure to fr...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>symptoms</td>\n",
              "      <td>What are the symptoms of Lymphocytic Choriomen...</td>\n",
              "      <td>LCMV is most commonly recognized as causing ne...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>susceptibility</td>\n",
              "      <td>Who is at risk for Lymphocytic Choriomeningiti...</td>\n",
              "      <td>Individuals of all ages who come into contact ...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>exams and tests</td>\n",
              "      <td>How to diagnose Lymphocytic Choriomeningitis (...</td>\n",
              "      <td>During the first phase of the disease, the mos...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>treatment</td>\n",
              "      <td>What are the treatments for Lymphocytic Chorio...</td>\n",
              "      <td>Aseptic meningitis, encephalitis, or meningoen...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>5</th>\n",
              "      <td>prevention</td>\n",
              "      <td>How to prevent Lymphocytic Choriomeningitis (L...</td>\n",
              "      <td>LCMV infection can be prevented by avoiding co...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>6</th>\n",
              "      <td>information</td>\n",
              "      <td>What is (are) Parasites - Cysticercosis ?</td>\n",
              "      <td>Cysticercosis is an infection caused by the la...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>7</th>\n",
              "      <td>susceptibility</td>\n",
              "      <td>Who is at risk for Parasites - Cysticercosis? ?</td>\n",
              "      <td>Cysticercosis is an infection caused by the la...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>8</th>\n",
              "      <td>exams and tests</td>\n",
              "      <td>How to diagnose Parasites - Cysticercosis ?</td>\n",
              "      <td>If you think that you may have cysticercosis, ...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>9</th>\n",
              "      <td>treatment</td>\n",
              "      <td>What are the treatments for Parasites - Cystic...</td>\n",
              "      <td>Some people with cysticercosis do not need to ...</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>\n",
              "    <div class=\"colab-df-buttons\">\n",
              "\n",
              "  <div class=\"colab-df-container\">\n",
              "    <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-c60464df-e98f-42d2-8b01-1aafc8d5698d')\"\n",
              "            title=\"Convert this dataframe to an interactive table.\"\n",
              "            style=\"display:none;\">\n",
              "\n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\" viewBox=\"0 -960 960 960\">\n",
              "    <path d=\"M120-120v-720h720v720H120Zm60-500h600v-160H180v160Zm220 220h160v-160H400v160Zm0 220h160v-160H400v160ZM180-400h160v-160H180v160Zm440 0h160v-160H620v160ZM180-180h160v-160H180v160Zm440 0h160v-160H620v160Z\"/>\n",
              "  </svg>\n",
              "    </button>\n",
              "\n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    .colab-df-buttons div {\n",
              "      margin-bottom: 4px;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "    <script>\n",
              "      const buttonEl =\n",
              "        document.querySelector('#df-c60464df-e98f-42d2-8b01-1aafc8d5698d button.colab-df-convert');\n",
              "      buttonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "      async function convertToInteractive(key) {\n",
              "        const element = document.querySelector('#df-c60464df-e98f-42d2-8b01-1aafc8d5698d');\n",
              "        const dataTable =\n",
              "          await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                    [key], {});\n",
              "        if (!dataTable) return;\n",
              "\n",
              "        const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "          '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "          + ' to learn more about interactive tables.';\n",
              "        element.innerHTML = '';\n",
              "        dataTable['output_type'] = 'display_data';\n",
              "        await google.colab.output.renderOutput(dataTable, element);\n",
              "        const docLink = document.createElement('div');\n",
              "        docLink.innerHTML = docLinkHtml;\n",
              "        element.appendChild(docLink);\n",
              "      }\n",
              "    </script>\n",
              "  </div>\n",
              "\n",
              "\n",
              "<div id=\"df-82590a6f-d889-4a9a-bdb9-525fa83b54b8\">\n",
              "  <button class=\"colab-df-quickchart\" onclick=\"quickchart('df-82590a6f-d889-4a9a-bdb9-525fa83b54b8')\"\n",
              "            title=\"Suggest charts\"\n",
              "            style=\"display:none;\">\n",
              "\n",
              "<svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "     width=\"24px\">\n",
              "    <g>\n",
              "        <path d=\"M19 3H5c-1.1 0-2 .9-2 2v14c0 1.1.9 2 2 2h14c1.1 0 2-.9 2-2V5c0-1.1-.9-2-2-2zM9 17H7v-7h2v7zm4 0h-2V7h2v10zm4 0h-2v-4h2v4z\"/>\n",
              "    </g>\n",
              "</svg>\n",
              "  </button>\n",
              "\n",
              "<style>\n",
              "  .colab-df-quickchart {\n",
              "      --bg-color: #E8F0FE;\n",
              "      --fill-color: #1967D2;\n",
              "      --hover-bg-color: #E2EBFA;\n",
              "      --hover-fill-color: #174EA6;\n",
              "      --disabled-fill-color: #AAA;\n",
              "      --disabled-bg-color: #DDD;\n",
              "  }\n",
              "\n",
              "  [theme=dark] .colab-df-quickchart {\n",
              "      --bg-color: #3B4455;\n",
              "      --fill-color: #D2E3FC;\n",
              "      --hover-bg-color: #434B5C;\n",
              "      --hover-fill-color: #FFFFFF;\n",
              "      --disabled-bg-color: #3B4455;\n",
              "      --disabled-fill-color: #666;\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart {\n",
              "    background-color: var(--bg-color);\n",
              "    border: none;\n",
              "    border-radius: 50%;\n",
              "    cursor: pointer;\n",
              "    display: none;\n",
              "    fill: var(--fill-color);\n",
              "    height: 32px;\n",
              "    padding: 0;\n",
              "    width: 32px;\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart:hover {\n",
              "    background-color: var(--hover-bg-color);\n",
              "    box-shadow: 0 1px 2px rgba(60, 64, 67, 0.3), 0 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "    fill: var(--button-hover-fill-color);\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart-complete:disabled,\n",
              "  .colab-df-quickchart-complete:disabled:hover {\n",
              "    background-color: var(--disabled-bg-color);\n",
              "    fill: var(--disabled-fill-color);\n",
              "    box-shadow: none;\n",
              "  }\n",
              "\n",
              "  .colab-df-spinner {\n",
              "    border: 2px solid var(--fill-color);\n",
              "    border-color: transparent;\n",
              "    border-bottom-color: var(--fill-color);\n",
              "    animation:\n",
              "      spin 1s steps(1) infinite;\n",
              "  }\n",
              "\n",
              "  @keyframes spin {\n",
              "    0% {\n",
              "      border-color: transparent;\n",
              "      border-bottom-color: var(--fill-color);\n",
              "      border-left-color: var(--fill-color);\n",
              "    }\n",
              "    20% {\n",
              "      border-color: transparent;\n",
              "      border-left-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "    }\n",
              "    30% {\n",
              "      border-color: transparent;\n",
              "      border-left-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "      border-right-color: var(--fill-color);\n",
              "    }\n",
              "    40% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "    }\n",
              "    60% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "    }\n",
              "    80% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "      border-bottom-color: var(--fill-color);\n",
              "    }\n",
              "    90% {\n",
              "      border-color: transparent;\n",
              "      border-bottom-color: var(--fill-color);\n",
              "    }\n",
              "  }\n",
              "</style>\n",
              "\n",
              "  <script>\n",
              "    async function quickchart(key) {\n",
              "      const quickchartButtonEl =\n",
              "        document.querySelector('#' + key + ' button');\n",
              "      quickchartButtonEl.disabled = true;  // To prevent multiple clicks.\n",
              "      quickchartButtonEl.classList.add('colab-df-spinner');\n",
              "      try {\n",
              "        const charts = await google.colab.kernel.invokeFunction(\n",
              "            'suggestCharts', [key], {});\n",
              "      } catch (error) {\n",
              "        console.error('Error during call to suggestCharts:', error);\n",
              "      }\n",
              "      quickchartButtonEl.classList.remove('colab-df-spinner');\n",
              "      quickchartButtonEl.classList.add('colab-df-quickchart-complete');\n",
              "    }\n",
              "    (() => {\n",
              "      let quickchartButtonEl =\n",
              "        document.querySelector('#df-82590a6f-d889-4a9a-bdb9-525fa83b54b8 button');\n",
              "      quickchartButtonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "    })();\n",
              "  </script>\n",
              "</div>\n",
              "    </div>\n",
              "  </div>\n"
            ],
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "dataframe",
              "variable_name": "data",
              "summary": "{\n  \"name\": \"data\",\n  \"rows\": 16407,\n  \"fields\": [\n    {\n      \"column\": \"qtype\",\n      \"properties\": {\n        \"dtype\": \"category\",\n        \"num_unique_values\": 16,\n        \"samples\": [\n          \"susceptibility\",\n          \"symptoms\",\n          \"information\"\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"Question\",\n      \"properties\": {\n        \"dtype\": \"string\",\n        \"num_unique_values\": 14979,\n        \"samples\": [\n          \"What are the symptoms of Danon disease ?\",\n          \"What is (are) Dowling-Degos disease ?\",\n          \"What are the genetic changes related to Pearson marrow-pancreas syndrome ?\"\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"Answer\",\n      \"properties\": {\n        \"dtype\": \"string\",\n        \"num_unique_values\": 15817,\n        \"samples\": [\n          \"These resources address the diagnosis or management of glycogen storage disease type III:  - Gene Review: Gene Review: Glycogen Storage Disease Type III  - Genetic Testing Registry: Glycogen storage disease type III   These resources from MedlinePlus offer information about the diagnosis and management of various health conditions:  - Diagnostic Tests  - Drug Therapy  - Surgery and Rehabilitation  - Genetic Counseling   - Palliative Care\",\n          \"Diagnostic Challenges\\n  \\nFor doctors, diagnosing chronic fatigue syndrome (CFS) can be complicated by a number of factors:\\n  \\n   - There's no lab test or biomarker for CFS.\\n   - Fatigue and other symptoms of CFS are common to many illnesses.\\n   - For some CFS patients, it may not be obvious to doctors that they are ill.\\n   - The illness has a pattern of remission and relapse.\\n   - Symptoms vary from person to person in type, number, and severity.\\n  \\n  \\nThese factors have contributed to a low diagnosis rate. Of the one to four million Americans who have CFS, less than 20% have been diagnosed.\\n  Exams and Screening Tests for CFS\\n  \\nBecause there is no blood test, brain scan, or other lab test to diagnose CFS, the doctor should first rule out other possible causes.\\n  \\nIf a patient has had 6 or more consecutive months of severe fatigue that is reported to be unrelieved by sufficient bed rest and that is accompanied by nonspecific symptoms, including flu-like symptoms, generalized pain, and memory problems, the doctor should consider the possibility that the patient may have CFS. Further exams and tests are needed before a diagnosis can be made:\\n  \\n   - A detailed medical history will be needed and should include a review of medications that could be causing the fatigue and symptoms\\n   - A thorough physical and mental status examination will also be needed\\n   - A battery of laboratory screening tests will be needed to help identify or rule out other possible causes of the symptoms that could be treated\\n   - The doctor may also order additional tests to follow up on results of the initial screening tests\\n  \\n  \\nA CFS diagnosis requires that the patient has been fatigued for 6 months or more and has 4 of the 8 symptoms for CFS for 6 months or more. If, however, the patient has been fatigued for 6 months or more but does not have four of the eight symptoms, the diagnosis may be idiopathic fatigue.\\n  \\nThe complete process for diagnosing CFS can be found here.\\n  \\nAdditional information for healthcare professionals on use of tests can be found here.\",\n          \"Eating, diet, and nutrition have not been shown to play a role in causing or preventing simple kidney cysts.\"\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    }\n  ]\n}"
            }
          },
          "metadata": {},
          "execution_count": 88
        }
      ],
      "source": [
        "data = data.to_pandas()\n",
        "data.head(10)"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "data = data[0:100]"
      ],
      "metadata": {
        "id": "hf7RQa6B5xpx"
      },
      "execution_count": 89,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "As you can see, the medical information in the dataset is well-organized, and to someone like me, who is not an expert in the field, it appears to be quite valuable. This information could be a useful addition to any general medicine book to support primary care doctors."
      ],
      "metadata": {
        "id": "4FhUslovtiqn"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "Load the langchain libraries to load the document."
      ],
      "metadata": {
        "id": "2sLkrHF6lWhM"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "from langchain.document_loaders import DataFrameLoader\n",
        "from langchain.vectorstores import Chroma"
      ],
      "metadata": {
        "id": "cCBAlIb596wZ"
      },
      "execution_count": 90,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "The Document is in the Answer column, and the others columns are Metadata."
      ],
      "metadata": {
        "id": "nGCkVX6xldOR"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "df_loader = DataFrameLoader(data, page_content_column=\"Answer\")\n"
      ],
      "metadata": {
        "id": "JZX8SaTe99Uf"
      },
      "execution_count": 91,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "df_document = df_loader.load()\n",
        "display(df_document[:2])"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 642
        },
        "id": "HDWnTqRY-IDr",
        "outputId": "45e56803-62a0-4d42-9889-244744196848"
      },
      "execution_count": 92,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "[Document(page_content='LCMV infections can occur after exposure to fresh urine, droppings, saliva, or nesting materials from infected rodents.  Transmission may also occur when these materials are directly introduced into broken skin, the nose, the eyes, or the mouth, or presumably, via the bite of an infected rodent. Person-to-person transmission has not been reported, with the exception of vertical transmission from infected mother to fetus, and rarely, through organ transplantation.', metadata={'qtype': 'susceptibility', 'Question': 'Who is at risk for Lymphocytic Choriomeningitis (LCM)? ?'}),\n",
              " Document(page_content='LCMV is most commonly recognized as causing neurological disease, as its name implies, though infection without symptoms or mild febrile illnesses are more common clinical manifestations. \\n                \\nFor infected persons who do become ill, onset of symptoms usually occurs 8-13 days after exposure to the virus as part of a biphasic febrile illness. This initial phase, which may last as long as a week, typically begins with any or all of the following symptoms: fever, malaise, lack of appetite, muscle aches, headache, nausea, and vomiting. Other symptoms appearing less frequently include sore throat, cough, joint pain, chest pain, testicular pain, and parotid (salivary gland) pain. \\n                \\nFollowing a few days of recovery, a second phase of illness may occur. Symptoms may consist of meningitis (fever, headache, stiff neck, etc.), encephalitis (drowsiness, confusion, sensory disturbances, and/or motor abnormalities, such as paralysis), or meningoencephalitis (inflammation of both the brain and meninges). LCMV has also been known to cause acute hydrocephalus (increased fluid on the brain), which often requires surgical shunting to relieve increased intracranial pressure. In rare instances, infection results in myelitis (inflammation of the spinal cord) and presents with symptoms such as muscle weakness, paralysis, or changes in body sensation. An association between LCMV infection and myocarditis (inflammation of the heart muscles) has been suggested. \\n                \\nPrevious observations show that most patients who develop aseptic meningitis or encephalitis due to LCMV survive. No chronic infection has been described in humans, and after the acute phase of illness, the virus is cleared from the body. However, as in all infections of the central nervous system, particularly encephalitis, temporary or permanent neurological damage is possible. Nerve deafness and arthritis have been reported. \\n                \\nWomen who become infected with LCMV during pregnancy may pass the infection on to the fetus. Infections occurring during the first trimester may result in fetal death and pregnancy termination, while in the second and third trimesters, birth defects can develop. Infants infected In utero can have many serious and permanent birth defects, including vision problems, mental retardation, and hydrocephaly (water on the brain). Pregnant women may recall a flu-like illness during pregnancy, or may not recall any illness. \\n                \\nLCM is usually not fatal. In general, mortality is less than 1%.', metadata={'qtype': 'symptoms', 'Question': 'What are the symptoms of Lymphocytic Choriomeningitis (LCM) ?'})]"
            ]
          },
          "metadata": {}
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "We can chunk the documents. The size to which we want to split the document is a design decision. The larger it is, the larger the prompt will be, and the slower the Model's response process.\n",
        "\n",
        "We also need to consider the maximum prompt size and ensure that the document does not exceed it."
      ],
      "metadata": {
        "id": "zPuTbSsXl3uF"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "from langchain.text_splitter import CharacterTextSplitter"
      ],
      "metadata": {
        "id": "8wogWV1_-lxh"
      },
      "execution_count": 93,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "text_splitter = CharacterTextSplitter(chunk_size=1250, chunk_overlap=100)\n",
        "texts = text_splitter.split_documents(df_document)\n"
      ],
      "metadata": {
        "id": "kg7SRkqO-f9x"
      },
      "execution_count": 94,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "These warnings we see are because it can't perform the partition of the required size. This is because it waits for a page break to divide the text and does so when possible."
      ],
      "metadata": {
        "id": "y0dn9EfrKoeW"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "first_doc = texts[1]\n",
        "print(first_doc.page_content)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "XaLWV_TzASSB",
        "outputId": "85ce0ec4-87d9-41c1-9386-0d905cb3fbca"
      },
      "execution_count": 95,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "LCMV is most commonly recognized as causing neurological disease, as its name implies, though infection without symptoms or mild febrile illnesses are more common clinical manifestations. \n",
            "                \n",
            "For infected persons who do become ill, onset of symptoms usually occurs 8-13 days after exposure to the virus as part of a biphasic febrile illness. This initial phase, which may last as long as a week, typically begins with any or all of the following symptoms: fever, malaise, lack of appetite, muscle aches, headache, nausea, and vomiting. Other symptoms appearing less frequently include sore throat, cough, joint pain, chest pain, testicular pain, and parotid (salivary gland) pain. \n",
            "                \n",
            "Following a few days of recovery, a second phase of illness may occur. Symptoms may consist of meningitis (fever, headache, stiff neck, etc.), encephalitis (drowsiness, confusion, sensory disturbances, and/or motor abnormalities, such as paralysis), or meningoencephalitis (inflammation of both the brain and meninges). LCMV has also been known to cause acute hydrocephalus (increased fluid on the brain), which often requires surgical shunting to relieve increased intracranial pressure. In rare instances, infection results in myelitis (inflammation of the spinal cord) and presents with symptoms such as muscle weakness, paralysis, or changes in body sensation. An association between LCMV infection and myocarditis (inflammation of the heart muscles) has been suggested. \n",
            "                \n",
            "Previous observations show that most patients who develop aseptic meningitis or encephalitis due to LCMV survive. No chronic infection has been described in humans, and after the acute phase of illness, the virus is cleared from the body. However, as in all infections of the central nervous system, particularly encephalitis, temporary or permanent neurological damage is possible. Nerve deafness and arthritis have been reported. \n",
            "                \n",
            "Women who become infected with LCMV during pregnancy may pass the infection on to the fetus. Infections occurring during the first trimester may result in fetal death and pregnancy termination, while in the second and third trimesters, birth defects can develop. Infants infected In utero can have many serious and permanent birth defects, including vision problems, mental retardation, and hydrocephaly (water on the brain). Pregnant women may recall a flu-like illness during pregnancy, or may not recall any illness. \n",
            "                \n",
            "LCM is usually not fatal. In general, mortality is less than 1%.\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "B2_Pt7N6Zg2X"
      },
      "source": [
        "### Initialize the Embedding Model and Vector DB"
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "We load the text-embedding-ada-002 model from OpenAI."
      ],
      "metadata": {
        "id": "n8ROb8oMnRLD"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "from getpass import getpass\n",
        "import os\n",
        "if not 'OPENAI_API_KEY' in os.environ:\n",
        "  os.environ[\"OPENAI_API_KEY\"] = getpass(\"OpenAI API Key: \")"
      ],
      "metadata": {
        "id": "F_Dn06xGwjKP"
      },
      "execution_count": 96,
      "outputs": []
    },
    {
      "cell_type": "code",
      "execution_count": 97,
      "metadata": {
        "id": "U57x2_87YSpb"
      },
      "outputs": [],
      "source": [
        "from langchain_openai import OpenAIEmbeddings\n",
        "\n",
        "model_name = 'text-embedding-ada-002'\n",
        "#model_name = 'text-embedding-3-small'\n",
        "\n",
        "embed = OpenAIEmbeddings(\n",
        "    model=model_name,\n",
        "    #openai_api_key=OPENAI_API_KEY\n",
        ")"
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "The execution of this cell may take 3 to 5 minutes. If you want it to be faster, you can reduce the number of records in the dataset."
      ],
      "metadata": {
        "id": "cgTCwF7UMyNW"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "directory_cdb = '/content/drive/MyDrive/chromadb'\n",
        "chroma_db = Chroma.from_documents(\n",
        "    df_document, embed, persist_directory=directory_cdb\n",
        ")"
      ],
      "metadata": {
        "id": "SEhQMQ8eCMj8"
      },
      "execution_count": 98,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "We are going to create three objects.\n",
        "\n",
        "* The language model, which can be any of those from OpenAI, the most common being gpt-3.5.\n",
        "* The memory, responsible for keeping the prompt with all the necessary history.\n",
        "* The retrieval, used to obtain information stored in ChromaDB."
      ],
      "metadata": {
        "id": "KhjV-T8GoarF"
      }
    },
    {
      "cell_type": "code",
      "execution_count": 126,
      "metadata": {
        "id": "zMRs9Klic5-Y"
      },
      "outputs": [],
      "source": [
        "from langchain.chat_models import ChatOpenAI\n",
        "from langchain_openai import OpenAI\n",
        "from langchain.chains.conversation.memory import ConversationBufferWindowMemory\n",
        "from langchain.chains import RetrievalQA\n",
        "\n",
        "llm=OpenAI(#openai_api_key=OPENAI_API_KEY,\n",
        "           temperature=0.0)\n",
        "\n",
        "conversational_memory = ConversationBufferWindowMemory(\n",
        "    memory_key='chat_history',\n",
        "    k=4, #Number of messages stored in memory\n",
        "    return_messages=True #Must return the messages in the response.\n",
        ")\n",
        "\n",
        "qa = RetrievalQA.from_chain_type(\n",
        "    llm=llm,\n",
        "    chain_type=\"stuff\",\n",
        "    retriever=chroma_db.as_retriever()\n",
        ")"
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "We can try the isolated Retrieval to see if the information it returns is relevant.\n",
        "\n",
        "\n"
      ],
      "metadata": {
        "id": "ergrieE4o8qu"
      }
    },
    {
      "cell_type": "code",
      "execution_count": 127,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 53
        },
        "id": "LaYSq0V-dxHw",
        "outputId": "63fdb5c9-fd35-4988-d48d-b76c15f69c21"
      },
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "' The main symptom of LCM is neurological disease, which can manifest as meningitis, encephalitis, or meningoencephalitis.'"
            ],
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "string"
            }
          },
          "metadata": {},
          "execution_count": 127
        }
      ],
      "source": [
        "qa.run(\"What is the main symptom of LCM?\")"
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "Perfect! The information returned is exactly what we desired.\n",
        "\n",
        "## Creating the Agent."
      ],
      "metadata": {
        "id": "pf9MXPeipEBO"
      }
    },
    {
      "cell_type": "code",
      "execution_count": 101,
      "metadata": {
        "id": "FwCYrS4duqBW"
      },
      "outputs": [],
      "source": [
        "from langchain.agents import Tool, AgentExecutor\n",
        "\n",
        "#Defining the list of tool objects to be used by LangChain.\n",
        "tools = [\n",
        "    Tool(\n",
        "        name='Medical KB',\n",
        "        func=qa.run,\n",
        "        description=(\n",
        "            'use this tool when answering medical knowledge queries to get '\n",
        "            'more information about the topic'\n",
        "        )\n",
        "    )\n",
        "]"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 102,
      "metadata": {
        "id": "JaKTzPUEvOoy"
      },
      "outputs": [],
      "source": [
        "from langchain.agents import initialize_agent, create_react_agent\n",
        "from langchain import hub\n",
        "\n",
        "prompt = hub.pull(\"hwchase17/react-chat\")\n",
        "agent = create_react_agent(\n",
        "    #agent='chat-conversational-react-description',\n",
        "    tools=tools,\n",
        "    llm=llm,\n",
        "    prompt=prompt\n",
        "    #verbose=True,\n",
        "    #max_iterations=3,\n",
        "    #early_stopping_method='generate',\n",
        "    #memory=conversational_memory\n",
        ")"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# Create an agent executor by passing in the agent and tools\n",
        "agent_executor = AgentExecutor(agent=agent,\n",
        "                               tools=tools,\n",
        "                               verbose=True,\n",
        "                               memory=conversational_memory,\n",
        "                               max_iterations=30,\n",
        "                               max_execution_time=600,\n",
        "                               #early_stopping_method='generate',\n",
        "                               handle_parsing_errors=True\n",
        "                               )"
      ],
      "metadata": {
        "id": "5sjTL1EU1vNW"
      },
      "execution_count": 128,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "IlxUBWKcvzeP"
      },
      "source": [
        "### Using the Conversational Agent"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "ZZapCP4Pv2kz"
      },
      "source": [
        "To make queries we simply call the `agent` directly.\n",
        "\n",
        "First i will try a order not related to the Medical field."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 104,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "85vipqC02deV",
        "outputId": "59548f2e-91d1-4b2a-c397-69ddd938c2d0"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "\n",
            "\u001b[1m> Entering new AgentExecutor chain...\u001b[0m\n",
            "\u001b[32;1m\u001b[1;3mThought: Do I need to use a tool? Yes\n",
            "Action: Medical KB\n",
            "Action Input: Area of square\u001b[0m\u001b[36;1m\u001b[1;3m I don't know.\u001b[0m\u001b[32;1m\u001b[1;3m Do I need to use a tool? No\n",
            "Final Answer: The area of a square with sides of 2 units is 4 square units.\u001b[0m\n",
            "\n",
            "\u001b[1m> Finished chain.\u001b[0m\n"
          ]
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "{'input': 'Give me the area of square of 2x2',\n",
              " 'chat_history': [],\n",
              " 'output': 'The area of a square with sides of 2 units is 4 square units.'}"
            ]
          },
          "metadata": {},
          "execution_count": 104
        }
      ],
      "source": [
        "agent_executor.invoke({\"input\": \"Give me the area of square of 2x2\"})"
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "Perfect, the model has responded without accessing the configured knowledge database.\n",
        "\n",
        "Now I will try with a question that is also not related to health."
      ],
      "metadata": {
        "id": "8suStMR7G11e"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "agent_executor.invoke({\"input\": \"Do you know who is Clark Kent?\"})"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "9YEsxEuCVgMv",
        "outputId": "5ea10363-9721-4d39-c238-854fb0c2ee72"
      },
      "execution_count": 105,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "\n",
            "\u001b[1m> Entering new AgentExecutor chain...\u001b[0m\n",
            "\u001b[32;1m\u001b[1;3m\n",
            "Thought: Do I need to use a tool? No\n",
            "Final Answer: Clark Kent is the secret identity of the superhero Superman.\u001b[0m\n",
            "\n",
            "\u001b[1m> Finished chain.\u001b[0m\n"
          ]
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "{'input': 'Do you know who is Clark Kent?',\n",
              " 'chat_history': [HumanMessage(content='Give me the area of square of 2x2'),\n",
              "  AIMessage(content='The area of a square with sides of 2 units is 4 square units.')],\n",
              " 'output': 'Clark Kent is the secret identity of the superhero Superman.'}"
            ]
          },
          "metadata": {},
          "execution_count": 105
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "It has not accessed either, as the model has been able to identify that it is not a question related to the database that LangChain provides.\n",
        "\n",
        "Now it's time to try with a question related to Medicine. Let's see if the model can understand that it should first look for information in the vector database at its disposal."
      ],
      "metadata": {
        "id": "lyd0XJ3CHjVb"
      }
    },
    {
      "cell_type": "code",
      "source": [
        " agent_executor.memory.clear()"
      ],
      "metadata": {
        "id": "Wtwgfuy158LV"
      },
      "execution_count": 106,
      "outputs": []
    },
    {
      "cell_type": "code",
      "execution_count": 107,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "RJoAhy76vzAB",
        "outputId": "277a9922-caac-4185-9cc0-15a61fcb11e6"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "\n",
            "\u001b[1m> Entering new AgentExecutor chain...\u001b[0m\n",
            "\u001b[32;1m\u001b[1;3m\n",
            "Thought: Do I need to use a tool? Yes\n",
            "Action: Medical KB\n",
            "Action Input: Botulism\u001b[0m\u001b[36;1m\u001b[1;3m Botulism is a rare but serious paralytic illness caused by a nerve toxin produced by certain bacteria. There are five main types of botulism, including foodborne, wound, infant, adult intestinal toxemia, and iatrogenic. All forms of botulism can be fatal and are considered medical emergencies. Foodborne botulism is a public health emergency because it can affect many people if they consume contaminated food.\u001b[0m\u001b[32;1m\u001b[1;3mDo I need to use a tool? No\n",
            "Final Answer: To confirm a diagnosis of botulism, a doctor may perform a physical exam, review symptoms and medical history, and order laboratory tests. These tests may include a stool or blood test to detect the presence of the botulinum toxin or the bacteria that produce it. Imaging tests, such as an MRI or CT scan, may also be used to look for signs of nerve damage. It is important to seek medical attention immediately if botulism is suspected, as early treatment can improve outcomes.\u001b[0m\n",
            "\n",
            "\u001b[1m> Finished chain.\u001b[0m\n"
          ]
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "{'input': 'I have a patient that can have Botulism,\\nhow can I confirm the diagnose?',\n",
              " 'chat_history': [],\n",
              " 'output': 'To confirm a diagnosis of botulism, a doctor may perform a physical exam, review symptoms and medical history, and order laboratory tests. These tests may include a stool or blood test to detect the presence of the botulinum toxin or the bacteria that produce it. Imaging tests, such as an MRI or CT scan, may also be used to look for signs of nerve damage. It is important to seek medical attention immediately if botulism is suspected, as early treatment can improve outcomes.'}"
            ]
          },
          "metadata": {},
          "execution_count": 107
        }
      ],
      "source": [
        "agent_executor.invoke({\"input\": \"\"\"I have a patient that can have Botulism,\n",
        "how can I confirm the diagnose?\"\"\"})"
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "Perfect, the most important thing for us is that it has been able to identify that it should go to the medical database to search for information about the symptoms."
      ],
      "metadata": {
        "id": "8lsaoF8nJNnR"
      }
    },
    {
      "cell_type": "code",
      "execution_count": 108,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "mQeicHTj2pmY",
        "outputId": "440daeda-d980-4d11-ea0e-bff5745035d3"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "\n",
            "\u001b[1m> Entering new AgentExecutor chain...\u001b[0m\n",
            "\u001b[32;1m\u001b[1;3m\n",
            "Thought: Do I need to use a tool? No\n",
            "Final Answer: Yes, botulism is a serious illness that can lead to paralysis and even death if left untreated. It is important to seek medical attention immediately if you suspect you or someone else may have botulism.\u001b[0m\n",
            "\n",
            "\u001b[1m> Finished chain.\u001b[0m\n"
          ]
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "{'input': 'Is this an important illness?',\n",
              " 'chat_history': [HumanMessage(content='I have a patient that can have Botulism,\\nhow can I confirm the diagnose?'),\n",
              "  AIMessage(content='To confirm a diagnosis of botulism, a doctor may perform a physical exam, review symptoms and medical history, and order laboratory tests. These tests may include a stool or blood test to detect the presence of the botulinum toxin or the bacteria that produce it. Imaging tests, such as an MRI or CT scan, may also be used to look for signs of nerve damage. It is important to seek medical attention immediately if botulism is suspected, as early treatment can improve outcomes.')],\n",
              " 'output': 'Yes, botulism is a serious illness that can lead to paralysis and even death if left untreated. It is important to seek medical attention immediately if you suspect you or someone else may have botulism.'}"
            ]
          },
          "metadata": {},
          "execution_count": 108
        }
      ],
      "source": [
        "agent_executor.invoke({\"input\": \"Is this an important illness?\"})"
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "And the memory works perfectly. We can maintain a conversation, taking into account that the model knows the previous questions and answers."
      ],
      "metadata": {
        "id": "quf3qyI_aVz3"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Evaluating the solution with Giskard\n",
        "\n",
        "Install and Load the libraries."
      ],
      "metadata": {
        "id": "FQwHcAMi3e85"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "!pip install -q giskard[llm]\n",
        "from giskard.rag import KnowledgeBase, generate_testset, evaluate"
      ],
      "metadata": {
        "id": "6Em-qkfK637x"
      },
      "execution_count": 109,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "Is necesary to create a Dataframe with just the column containing the information used to create the RAG system."
      ],
      "metadata": {
        "id": "qXN4hGqDoMg1"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "import pandas as pd\n",
        "df_giskard = pd.DataFrame([d.page_content for d in df_document], columns=[\"text\"])\n",
        "df_giskard.head(10)"
      ],
      "metadata": {
        "id": "yeIMrHlhMJDF",
        "outputId": "8df35d55-9231-4c81-940a-7fbe03c3f367",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 363
        }
      },
      "execution_count": 112,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "                                                text\n",
              "0  LCMV infections can occur after exposure to fr...\n",
              "1  LCMV is most commonly recognized as causing ne...\n",
              "2  Individuals of all ages who come into contact ...\n",
              "3  During the first phase of the disease, the mos...\n",
              "4  Aseptic meningitis, encephalitis, or meningoen...\n",
              "5  LCMV infection can be prevented by avoiding co...\n",
              "6  Cysticercosis is an infection caused by the la...\n",
              "7  Cysticercosis is an infection caused by the la...\n",
              "8  If you think that you may have cysticercosis, ...\n",
              "9  Some people with cysticercosis do not need to ..."
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-77e8edc9-e7d2-488c-8fb0-fb789c4ad623\" class=\"colab-df-container\">\n",
              "    <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>text</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>LCMV infections can occur after exposure to fr...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>LCMV is most commonly recognized as causing ne...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>Individuals of all ages who come into contact ...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>During the first phase of the disease, the mos...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>Aseptic meningitis, encephalitis, or meningoen...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>5</th>\n",
              "      <td>LCMV infection can be prevented by avoiding co...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>6</th>\n",
              "      <td>Cysticercosis is an infection caused by the la...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>7</th>\n",
              "      <td>Cysticercosis is an infection caused by the la...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>8</th>\n",
              "      <td>If you think that you may have cysticercosis, ...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>9</th>\n",
              "      <td>Some people with cysticercosis do not need to ...</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>\n",
              "    <div class=\"colab-df-buttons\">\n",
              "\n",
              "  <div class=\"colab-df-container\">\n",
              "    <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-77e8edc9-e7d2-488c-8fb0-fb789c4ad623')\"\n",
              "            title=\"Convert this dataframe to an interactive table.\"\n",
              "            style=\"display:none;\">\n",
              "\n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\" viewBox=\"0 -960 960 960\">\n",
              "    <path d=\"M120-120v-720h720v720H120Zm60-500h600v-160H180v160Zm220 220h160v-160H400v160Zm0 220h160v-160H400v160ZM180-400h160v-160H180v160Zm440 0h160v-160H620v160ZM180-180h160v-160H180v160Zm440 0h160v-160H620v160Z\"/>\n",
              "  </svg>\n",
              "    </button>\n",
              "\n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    .colab-df-buttons div {\n",
              "      margin-bottom: 4px;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "    <script>\n",
              "      const buttonEl =\n",
              "        document.querySelector('#df-77e8edc9-e7d2-488c-8fb0-fb789c4ad623 button.colab-df-convert');\n",
              "      buttonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "      async function convertToInteractive(key) {\n",
              "        const element = document.querySelector('#df-77e8edc9-e7d2-488c-8fb0-fb789c4ad623');\n",
              "        const dataTable =\n",
              "          await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                    [key], {});\n",
              "        if (!dataTable) return;\n",
              "\n",
              "        const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "          '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "          + ' to learn more about interactive tables.';\n",
              "        element.innerHTML = '';\n",
              "        dataTable['output_type'] = 'display_data';\n",
              "        await google.colab.output.renderOutput(dataTable, element);\n",
              "        const docLink = document.createElement('div');\n",
              "        docLink.innerHTML = docLinkHtml;\n",
              "        element.appendChild(docLink);\n",
              "      }\n",
              "    </script>\n",
              "  </div>\n",
              "\n",
              "\n",
              "<div id=\"df-67e7f29f-bf22-401c-a2c6-7075fab8065a\">\n",
              "  <button class=\"colab-df-quickchart\" onclick=\"quickchart('df-67e7f29f-bf22-401c-a2c6-7075fab8065a')\"\n",
              "            title=\"Suggest charts\"\n",
              "            style=\"display:none;\">\n",
              "\n",
              "<svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "     width=\"24px\">\n",
              "    <g>\n",
              "        <path d=\"M19 3H5c-1.1 0-2 .9-2 2v14c0 1.1.9 2 2 2h14c1.1 0 2-.9 2-2V5c0-1.1-.9-2-2-2zM9 17H7v-7h2v7zm4 0h-2V7h2v10zm4 0h-2v-4h2v4z\"/>\n",
              "    </g>\n",
              "</svg>\n",
              "  </button>\n",
              "\n",
              "<style>\n",
              "  .colab-df-quickchart {\n",
              "      --bg-color: #E8F0FE;\n",
              "      --fill-color: #1967D2;\n",
              "      --hover-bg-color: #E2EBFA;\n",
              "      --hover-fill-color: #174EA6;\n",
              "      --disabled-fill-color: #AAA;\n",
              "      --disabled-bg-color: #DDD;\n",
              "  }\n",
              "\n",
              "  [theme=dark] .colab-df-quickchart {\n",
              "      --bg-color: #3B4455;\n",
              "      --fill-color: #D2E3FC;\n",
              "      --hover-bg-color: #434B5C;\n",
              "      --hover-fill-color: #FFFFFF;\n",
              "      --disabled-bg-color: #3B4455;\n",
              "      --disabled-fill-color: #666;\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart {\n",
              "    background-color: var(--bg-color);\n",
              "    border: none;\n",
              "    border-radius: 50%;\n",
              "    cursor: pointer;\n",
              "    display: none;\n",
              "    fill: var(--fill-color);\n",
              "    height: 32px;\n",
              "    padding: 0;\n",
              "    width: 32px;\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart:hover {\n",
              "    background-color: var(--hover-bg-color);\n",
              "    box-shadow: 0 1px 2px rgba(60, 64, 67, 0.3), 0 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "    fill: var(--button-hover-fill-color);\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart-complete:disabled,\n",
              "  .colab-df-quickchart-complete:disabled:hover {\n",
              "    background-color: var(--disabled-bg-color);\n",
              "    fill: var(--disabled-fill-color);\n",
              "    box-shadow: none;\n",
              "  }\n",
              "\n",
              "  .colab-df-spinner {\n",
              "    border: 2px solid var(--fill-color);\n",
              "    border-color: transparent;\n",
              "    border-bottom-color: var(--fill-color);\n",
              "    animation:\n",
              "      spin 1s steps(1) infinite;\n",
              "  }\n",
              "\n",
              "  @keyframes spin {\n",
              "    0% {\n",
              "      border-color: transparent;\n",
              "      border-bottom-color: var(--fill-color);\n",
              "      border-left-color: var(--fill-color);\n",
              "    }\n",
              "    20% {\n",
              "      border-color: transparent;\n",
              "      border-left-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "    }\n",
              "    30% {\n",
              "      border-color: transparent;\n",
              "      border-left-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "      border-right-color: var(--fill-color);\n",
              "    }\n",
              "    40% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "    }\n",
              "    60% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "    }\n",
              "    80% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "      border-bottom-color: var(--fill-color);\n",
              "    }\n",
              "    90% {\n",
              "      border-color: transparent;\n",
              "      border-bottom-color: var(--fill-color);\n",
              "    }\n",
              "  }\n",
              "</style>\n",
              "\n",
              "  <script>\n",
              "    async function quickchart(key) {\n",
              "      const quickchartButtonEl =\n",
              "        document.querySelector('#' + key + ' button');\n",
              "      quickchartButtonEl.disabled = true;  // To prevent multiple clicks.\n",
              "      quickchartButtonEl.classList.add('colab-df-spinner');\n",
              "      try {\n",
              "        const charts = await google.colab.kernel.invokeFunction(\n",
              "            'suggestCharts', [key], {});\n",
              "      } catch (error) {\n",
              "        console.error('Error during call to suggestCharts:', error);\n",
              "      }\n",
              "      quickchartButtonEl.classList.remove('colab-df-spinner');\n",
              "      quickchartButtonEl.classList.add('colab-df-quickchart-complete');\n",
              "    }\n",
              "    (() => {\n",
              "      let quickchartButtonEl =\n",
              "        document.querySelector('#df-67e7f29f-bf22-401c-a2c6-7075fab8065a button');\n",
              "      quickchartButtonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "    })();\n",
              "  </script>\n",
              "</div>\n",
              "    </div>\n",
              "  </div>\n"
            ],
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "dataframe",
              "variable_name": "df_giskard",
              "summary": "{\n  \"name\": \"df_giskard\",\n  \"rows\": 100,\n  \"fields\": [\n    {\n      \"column\": \"text\",\n      \"properties\": {\n        \"dtype\": \"string\",\n        \"num_unique_values\": 100,\n        \"samples\": [\n          \"The respiratory failure and paralysis that occur with severe botulism may require a patient to be on a breathing machine (ventilator) for weeks or months, plus intensive medical and nursing care. The paralysis slowly improves. Botulism can be treated with an antitoxin which blocks the action of toxin circulating in the blood. Antitoxin for infants is available from the California Department of Public Health, and antitoxin for older children and adults is available through CDC.If given before paralysis is complete, antitoxin can prevent worsening and shorten recovery time. Physicians may try to remove contaminated food still in the gut by inducing vomiting or by using enemas. Wounds should be treated, usually surgically, to remove the source of the toxin-producing bacteria followed by administration of appropriate antibiotics. Good supportive care in a hospital is the mainstay of therapy for all forms of botulism.\",\n          \"Suggested General Guidelines \\n  \\n    \\n   \\n  \\nIt is important to remember that the first time a person gets scabies they usually have no symptoms during the first 2 to 6 weeks they are infested; however they can still spread scabies during this time. \\n  \\nTreatment should be given to both the infested person and to household members and sexual contacts, particularly those who have had prolonged direct skin-to-skin contact with the infested person. Both sexual and close personal contacts who have had direct prolonged skin-to-skin contact with an infested person within the preceding month should be examined and treated. All persons should be treated at the same time to prevent reinfestation. Scabies may sometimes be sexually-acquired in adults, but is rarely sexually-acquired in children. \\n  \\nBedding, clothing, and towels used by infested persons or their household, sexual, and close contacts (as defined above) anytime during the three days before treatment should be decontaminated by washing in hot water and drying in a hot dryer, by dry-cleaning, or by sealing in a plastic bag for at least 72 hours. Scabies mites generally do not survive more than 2 to 3 days away from human skin. \\n  \\nUse of insecticide sprays and fumigants is not recommended. \\n    \\n  Medications Used to Treat Scabies \\n  \\n    \\n   \\n  \\nProducts used to treat scabies are called scabicides because they kill scabies mites; some also kill mite eggs. Scabicides used to treat human scabies are available only with a doctor\\u2019s prescription. No \\u201cover-the-counter\\u201d (non-prescription) products have been tested and approved to treat scabies. \\n  \\nScabicide should be applied to all areas of the body from the neck down to the feet and toes. In addition, when treating infants and young children, scabicide also should be applied to their entire head and neck because scabies can affect their face, scalp, and neck, as well as the rest of their body. The scabicide should be applied to a clean body and left on for the recommended time before washing it off. Clean clothing should be worn after treatment. \\n  \\nThe instructions contained in the box or printed on the label always should be followed carefully. Always contact a doctor or pharmacist if unsure how to use a particular medicine. \\n  \\nBecause the symptoms of scabies are due to a hypersensitivity reaction (allergy) to mites and their feces (scybala), itching still may continue for several weeks after treatment even if all the mites and eggs are killed. If itching still is present more than 2 to 4 weeks after treatment or if new burrows or pimple-like rash lesions continue to appear, retreatment may be necessary. \\n  \\nSkin sores that become infected should be treated with an appropriate antibiotic prescribed by a doctor.\",\n          \"Baylisascaris infection can be prevented by avoiding contact with raccoons and their feces. Washing your hands after working or playing outdoors is good practice for preventing a number of diseases.\\n    \\nDo not keep, feed, or adopt wild animals, including raccoons, as pets. Infection rarely causes symptoms in raccoons, so you cannot tell if a raccoon is infected by observing its behavior. Roundworm eggs passed in the feces of infected raccoons are not visible to the naked eye. Eggs can only be seen using a microscope.\\n    \\n     \\n    \\n    \\nYou may discourage raccoons from living in and around your home or parks by taking these steps:\\n    \\n     - prevent access to food\\n     - keep trash containers tightly closed\\n     - close off access to attics and basements\\n     - keep sandboxes covered when not in use (raccoons may use sandboxes as a latrine)\\n     - remove fish ponds -- they eat the fish and drink the water\\n     - eliminate water sources\\n     - remove bird feeders\\n     - clear brush so raccoons are not likely to make a den on your property\\n    \\n    \\nStay away from areas and materials that might be contaminated by raccoon feces. Raccoons typically defecate at the base of or in raised forks of trees, or on raised horizontal surfaces such as fallen logs, stumps, or large rocks. Raccoon feces also can be found on woodpiles, decks, rooftops, and in attics, garages, and haylofts. Feces usually are dark and tubular, have a pungent odor (usually worse than dog or cat feces), and often contain undigested seeds or other food items.\\n    \\nIf you have found a raccoon latrine near your home, cleaning the area may prevent possible infection. Newly deposited eggs take at least 2-4 weeks to become infective. Prompt removal and destruction of raccoon feces will reduce risk for exposure and possible infection.\\n    \\nMore on: Raccoon Latrine Clean-up [PDF, 111 KB, 1 page]\\n    \\n     \\n    \\n    \\nIf you choose to clean the site yourself, care should be taken to avoid contaminating hands and clothes.\\n    \\n     - Wear disposable gloves to help prevent cross contamination.\\n     - Wear a N95-rated respirator if working in a confined space to prevent accidental ingestion of eggs or other harmful materials.\\n     - Avoid stirring up dust and debris- you can lightly mist the latrine area with a little water from a spray bottle to reduce the amount of dust.\\n     - Wear rubber boots that can be scrubbed or cover your shoes with disposable booties that can be thrown away, so that you do not bring eggs into your household.\\n     - Feces and material contaminated with raccoon feces should be removed and burned, buried, or sent to a landfill.\\n     - Most chemicals do not kill roundworm eggs; however, heat kills the eggs instantly.\\n     - Treat feces-soiled decks, patios, and other surfaces with boiling water or a propane torch (please contact your local fire department for regulations and safety practices).\\n    \\n    \\nTo help further reduce the risk of possible infection, wash your hands well with soap and warm running water. Clean/launder your clothes thoroughly with hot water and detergent.\\n    \\nMore on: Handwashing\\n    \\nIf you are cleaning an indoor raccoon latrine and are not able to use a propane torch, use a damp (but not wet) sponge to wipe the area with hot soapy water. Rinse your sponge frequently. After you are finished, flush dirty water down the toilet. Place the sponge in a plastic bag and put the plastic bag in the garbage.\\n    \\nContact your local animal control office for additional assistance.\\n    Dogs\\n    \\nDogs may be infected with adult B. procyonis roundworms, but may not show symptoms. Have all pets de-wormed under a veterinarian's supervision and take precautions to avoid contact with their feces.\\n    Exotic pets\\n    \\nRaccoons and dogs are not the only hosts of Baylisascaris. B. procyonis infection has also been documented in kinkajous. Other animals such as coatis may be susceptible. When wild animals are kept as pets, there can be a risk of disease transmission to humans.\"\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    }\n  ]\n}"
            }
          },
          "metadata": {},
          "execution_count": 112
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "Using the information from the dataset, we ask Giskard to create a Knowledge Base, which is nothing more than a set of questions along with their respective answers. Both the questions and answers are generated by OpenAI's most advanced model, which is why it requires our OpenAI key to be provided."
      ],
      "metadata": {
        "id": "VK3hgB-MoWLj"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "kb_giskard = KnowledgeBase(df_giskard)"
      ],
      "metadata": {
        "id": "iPaw4w9c_Gbo"
      },
      "execution_count": 113,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# The more questions you generate, the more you will be charged.\n",
        "test_questions = generate_testset(\n",
        "    kb_giskard,\n",
        "    num_questions=30,\n",
        "    agent_description=\"Medical assistant for diagnosis and treatment support.\",\n",
        ")"
      ],
      "metadata": {
        "id": "VcSnDxElKQmw",
        "outputId": "c19c5b13-e1be-4e4d-ab83-865fe2f8dc39",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 49,
          "referenced_widgets": [
            "a1375f0fef4d43318ce4d893fe00cee8",
            "047b1132836c43f789ee3ffa794fb78a",
            "79de9c8407214fe5b6336a06e4bb0349",
            "816cd7664bd143dba6cadc7e44ee1eb1",
            "0d5d96c8d3d44f789bb6a01b658c1d2a",
            "df6477401b054940900648bfddbb8fa8",
            "e24cd9cfbd3347fd8efab530ff2c035f",
            "44c19c4eed39471695be2866ae2bbc16",
            "5794c634f988490c819e1cca2b8fb3d9",
            "f461e8947b584ec8a9ea0d8e190d5f34",
            "a85754523d064103863f4e5bc78afcc1"
          ]
        }
      },
      "execution_count": 131,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "Generating questions:   0%|          | 0/30 [00:00<?, ?it/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "a1375f0fef4d43318ce4d893fe00cee8"
            }
          },
          "metadata": {}
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "df_test_questions = test_questions.to_pandas()"
      ],
      "metadata": {
        "id": "rYH3J0MXTJl1"
      },
      "execution_count": 132,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "df_test_questions.head()"
      ],
      "metadata": {
        "id": "rvHhO_oDThft",
        "outputId": "5271f070-bbef-4772-b6b3-9befb5bf5e14",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 552
        }
      },
      "execution_count": 133,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "                                                                               question  \\\n",
              "id                                                                                        \n",
              "d4bb2ef9-c632-42a0-9335-666c0615cc8d  How can humans become infected with Baylisasca...   \n",
              "9985cfc1-6c89-47bd-9bc1-5dae2e3fc6d4  What are the steps to prevent and control the ...   \n",
              "e4a9279e-71ba-4beb-8bf2-23f345ee0d2a  What is the recommended prevention method for ...   \n",
              "79cb5115-abfc-4794-87e1-9ff9de886b2e                What is the treatment for botulism?   \n",
              "93d7f209-4f14-4bd1-bc54-9011e5229ada  What are the first symptoms of rabies and how ...   \n",
              "\n",
              "                                                                       reference_answer  \\\n",
              "id                                                                                        \n",
              "d4bb2ef9-c632-42a0-9335-666c0615cc8d  Humans become infected by ingesting embryonate...   \n",
              "9985cfc1-6c89-47bd-9bc1-5dae2e3fc6d4  The steps to prevent and control the spread of...   \n",
              "e4a9279e-71ba-4beb-8bf2-23f345ee0d2a  Scabies is prevented by avoiding direct skin-t...   \n",
              "79cb5115-abfc-4794-87e1-9ff9de886b2e  Botulism can be treated with an antitoxin whic...   \n",
              "93d7f209-4f14-4bd1-bc54-9011e5229ada  The first symptoms of rabies may be very simil...   \n",
              "\n",
              "                                                                      reference_context  \\\n",
              "id                                                                                        \n",
              "d4bb2ef9-c632-42a0-9335-666c0615cc8d  Document 70: Baylisascaris infection can be pr...   \n",
              "9985cfc1-6c89-47bd-9bc1-5dae2e3fc6d4  Document 34: Body lice are parasitic insects t...   \n",
              "e4a9279e-71ba-4beb-8bf2-23f345ee0d2a  Document 51: Transmission\\n    \\nHuman scabies...   \n",
              "79cb5115-abfc-4794-87e1-9ff9de886b2e  Document 81: The classic symptoms of botulism ...   \n",
              "93d7f209-4f14-4bd1-bc54-9011e5229ada  Document 76: The first symptoms of rabies may ...   \n",
              "\n",
              "                                     conversation_history  \\\n",
              "id                                                          \n",
              "d4bb2ef9-c632-42a0-9335-666c0615cc8d                   []   \n",
              "9985cfc1-6c89-47bd-9bc1-5dae2e3fc6d4                   []   \n",
              "e4a9279e-71ba-4beb-8bf2-23f345ee0d2a                   []   \n",
              "79cb5115-abfc-4794-87e1-9ff9de886b2e                   []   \n",
              "93d7f209-4f14-4bd1-bc54-9011e5229ada                   []   \n",
              "\n",
              "                                                                               metadata  \n",
              "id                                                                                       \n",
              "d4bb2ef9-c632-42a0-9335-666c0615cc8d  {'question_type': 'simple', 'seed_document_id'...  \n",
              "9985cfc1-6c89-47bd-9bc1-5dae2e3fc6d4  {'question_type': 'simple', 'seed_document_id'...  \n",
              "e4a9279e-71ba-4beb-8bf2-23f345ee0d2a  {'question_type': 'simple', 'seed_document_id'...  \n",
              "79cb5115-abfc-4794-87e1-9ff9de886b2e  {'question_type': 'simple', 'seed_document_id'...  \n",
              "93d7f209-4f14-4bd1-bc54-9011e5229ada  {'question_type': 'simple', 'seed_document_id'...  "
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-a64883fa-1cc2-4605-adf1-9212eae3a636\" class=\"colab-df-container\">\n",
              "    <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>question</th>\n",
              "      <th>reference_answer</th>\n",
              "      <th>reference_context</th>\n",
              "      <th>conversation_history</th>\n",
              "      <th>metadata</th>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>id</th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>d4bb2ef9-c632-42a0-9335-666c0615cc8d</th>\n",
              "      <td>How can humans become infected with Baylisasca...</td>\n",
              "      <td>Humans become infected by ingesting embryonate...</td>\n",
              "      <td>Document 70: Baylisascaris infection can be pr...</td>\n",
              "      <td>[]</td>\n",
              "      <td>{'question_type': 'simple', 'seed_document_id'...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>9985cfc1-6c89-47bd-9bc1-5dae2e3fc6d4</th>\n",
              "      <td>What are the steps to prevent and control the ...</td>\n",
              "      <td>The steps to prevent and control the spread of...</td>\n",
              "      <td>Document 34: Body lice are parasitic insects t...</td>\n",
              "      <td>[]</td>\n",
              "      <td>{'question_type': 'simple', 'seed_document_id'...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>e4a9279e-71ba-4beb-8bf2-23f345ee0d2a</th>\n",
              "      <td>What is the recommended prevention method for ...</td>\n",
              "      <td>Scabies is prevented by avoiding direct skin-t...</td>\n",
              "      <td>Document 51: Transmission\\n    \\nHuman scabies...</td>\n",
              "      <td>[]</td>\n",
              "      <td>{'question_type': 'simple', 'seed_document_id'...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>79cb5115-abfc-4794-87e1-9ff9de886b2e</th>\n",
              "      <td>What is the treatment for botulism?</td>\n",
              "      <td>Botulism can be treated with an antitoxin whic...</td>\n",
              "      <td>Document 81: The classic symptoms of botulism ...</td>\n",
              "      <td>[]</td>\n",
              "      <td>{'question_type': 'simple', 'seed_document_id'...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>93d7f209-4f14-4bd1-bc54-9011e5229ada</th>\n",
              "      <td>What are the first symptoms of rabies and how ...</td>\n",
              "      <td>The first symptoms of rabies may be very simil...</td>\n",
              "      <td>Document 76: The first symptoms of rabies may ...</td>\n",
              "      <td>[]</td>\n",
              "      <td>{'question_type': 'simple', 'seed_document_id'...</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>\n",
              "    <div class=\"colab-df-buttons\">\n",
              "\n",
              "  <div class=\"colab-df-container\">\n",
              "    <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-a64883fa-1cc2-4605-adf1-9212eae3a636')\"\n",
              "            title=\"Convert this dataframe to an interactive table.\"\n",
              "            style=\"display:none;\">\n",
              "\n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\" viewBox=\"0 -960 960 960\">\n",
              "    <path d=\"M120-120v-720h720v720H120Zm60-500h600v-160H180v160Zm220 220h160v-160H400v160Zm0 220h160v-160H400v160ZM180-400h160v-160H180v160Zm440 0h160v-160H620v160ZM180-180h160v-160H180v160Zm440 0h160v-160H620v160Z\"/>\n",
              "  </svg>\n",
              "    </button>\n",
              "\n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    .colab-df-buttons div {\n",
              "      margin-bottom: 4px;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "    <script>\n",
              "      const buttonEl =\n",
              "        document.querySelector('#df-a64883fa-1cc2-4605-adf1-9212eae3a636 button.colab-df-convert');\n",
              "      buttonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "      async function convertToInteractive(key) {\n",
              "        const element = document.querySelector('#df-a64883fa-1cc2-4605-adf1-9212eae3a636');\n",
              "        const dataTable =\n",
              "          await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                    [key], {});\n",
              "        if (!dataTable) return;\n",
              "\n",
              "        const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "          '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "          + ' to learn more about interactive tables.';\n",
              "        element.innerHTML = '';\n",
              "        dataTable['output_type'] = 'display_data';\n",
              "        await google.colab.output.renderOutput(dataTable, element);\n",
              "        const docLink = document.createElement('div');\n",
              "        docLink.innerHTML = docLinkHtml;\n",
              "        element.appendChild(docLink);\n",
              "      }\n",
              "    </script>\n",
              "  </div>\n",
              "\n",
              "\n",
              "<div id=\"df-2ea9b623-fe98-45be-bb89-22e6f3324981\">\n",
              "  <button class=\"colab-df-quickchart\" onclick=\"quickchart('df-2ea9b623-fe98-45be-bb89-22e6f3324981')\"\n",
              "            title=\"Suggest charts\"\n",
              "            style=\"display:none;\">\n",
              "\n",
              "<svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "     width=\"24px\">\n",
              "    <g>\n",
              "        <path d=\"M19 3H5c-1.1 0-2 .9-2 2v14c0 1.1.9 2 2 2h14c1.1 0 2-.9 2-2V5c0-1.1-.9-2-2-2zM9 17H7v-7h2v7zm4 0h-2V7h2v10zm4 0h-2v-4h2v4z\"/>\n",
              "    </g>\n",
              "</svg>\n",
              "  </button>\n",
              "\n",
              "<style>\n",
              "  .colab-df-quickchart {\n",
              "      --bg-color: #E8F0FE;\n",
              "      --fill-color: #1967D2;\n",
              "      --hover-bg-color: #E2EBFA;\n",
              "      --hover-fill-color: #174EA6;\n",
              "      --disabled-fill-color: #AAA;\n",
              "      --disabled-bg-color: #DDD;\n",
              "  }\n",
              "\n",
              "  [theme=dark] .colab-df-quickchart {\n",
              "      --bg-color: #3B4455;\n",
              "      --fill-color: #D2E3FC;\n",
              "      --hover-bg-color: #434B5C;\n",
              "      --hover-fill-color: #FFFFFF;\n",
              "      --disabled-bg-color: #3B4455;\n",
              "      --disabled-fill-color: #666;\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart {\n",
              "    background-color: var(--bg-color);\n",
              "    border: none;\n",
              "    border-radius: 50%;\n",
              "    cursor: pointer;\n",
              "    display: none;\n",
              "    fill: var(--fill-color);\n",
              "    height: 32px;\n",
              "    padding: 0;\n",
              "    width: 32px;\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart:hover {\n",
              "    background-color: var(--hover-bg-color);\n",
              "    box-shadow: 0 1px 2px rgba(60, 64, 67, 0.3), 0 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "    fill: var(--button-hover-fill-color);\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart-complete:disabled,\n",
              "  .colab-df-quickchart-complete:disabled:hover {\n",
              "    background-color: var(--disabled-bg-color);\n",
              "    fill: var(--disabled-fill-color);\n",
              "    box-shadow: none;\n",
              "  }\n",
              "\n",
              "  .colab-df-spinner {\n",
              "    border: 2px solid var(--fill-color);\n",
              "    border-color: transparent;\n",
              "    border-bottom-color: var(--fill-color);\n",
              "    animation:\n",
              "      spin 1s steps(1) infinite;\n",
              "  }\n",
              "\n",
              "  @keyframes spin {\n",
              "    0% {\n",
              "      border-color: transparent;\n",
              "      border-bottom-color: var(--fill-color);\n",
              "      border-left-color: var(--fill-color);\n",
              "    }\n",
              "    20% {\n",
              "      border-color: transparent;\n",
              "      border-left-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "    }\n",
              "    30% {\n",
              "      border-color: transparent;\n",
              "      border-left-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "      border-right-color: var(--fill-color);\n",
              "    }\n",
              "    40% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "    }\n",
              "    60% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "    }\n",
              "    80% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "      border-bottom-color: var(--fill-color);\n",
              "    }\n",
              "    90% {\n",
              "      border-color: transparent;\n",
              "      border-bottom-color: var(--fill-color);\n",
              "    }\n",
              "  }\n",
              "</style>\n",
              "\n",
              "  <script>\n",
              "    async function quickchart(key) {\n",
              "      const quickchartButtonEl =\n",
              "        document.querySelector('#' + key + ' button');\n",
              "      quickchartButtonEl.disabled = true;  // To prevent multiple clicks.\n",
              "      quickchartButtonEl.classList.add('colab-df-spinner');\n",
              "      try {\n",
              "        const charts = await google.colab.kernel.invokeFunction(\n",
              "            'suggestCharts', [key], {});\n",
              "      } catch (error) {\n",
              "        console.error('Error during call to suggestCharts:', error);\n",
              "      }\n",
              "      quickchartButtonEl.classList.remove('colab-df-spinner');\n",
              "      quickchartButtonEl.classList.add('colab-df-quickchart-complete');\n",
              "    }\n",
              "    (() => {\n",
              "      let quickchartButtonEl =\n",
              "        document.querySelector('#df-2ea9b623-fe98-45be-bb89-22e6f3324981 button');\n",
              "      quickchartButtonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "    })();\n",
              "  </script>\n",
              "</div>\n",
              "    </div>\n",
              "  </div>\n"
            ],
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "dataframe",
              "variable_name": "df_test_questions",
              "summary": "{\n  \"name\": \"df_test_questions\",\n  \"rows\": 30,\n  \"fields\": [\n    {\n      \"column\": \"id\",\n      \"properties\": {\n        \"dtype\": \"string\",\n        \"num_unique_values\": 30,\n        \"samples\": [\n          \"fe987ee4-8c66-43c4-a5ba-3f3c42661557\",\n          \"c57f429d-3e39-4b97-84a6-cba54f5494cd\",\n          \"a48ec584-6c4e-42d5-9c9e-cdc227275968\"\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"question\",\n      \"properties\": {\n        \"dtype\": \"string\",\n        \"num_unique_values\": 30,\n        \"samples\": [\n          \"What are some preventive measures?\",\n          \"I am a worried mother and I suspect that my son might have contracted the LCMV virus. Can you please tell me what the potential consequences are if he contracted it during my pregnancy?\",\n          \"What are the five main kinds of botulism and what are their main causes in the United States?\"\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"reference_answer\",\n      \"properties\": {\n        \"dtype\": \"string\",\n        \"num_unique_values\": 30,\n        \"samples\": [\n          \"Some preventive measures to avoid head lice infestation include avoiding head-to-head contact during play and other activities, not sharing clothing such as hats, scarves, coats, sports uniforms, hair ribbons, or barrettes, not sharing combs, brushes, or towels and not lying on beds, couches, pillows, carpets, or stuffed animals that have recently been in contact with an infested person.\",\n          \"Women who become infected with LCMV during pregnancy may pass the infection on to the fetus. Infections occurring during the first trimester may result in fetal death and pregnancy termination, while in the second and third trimesters, birth defects can develop. Infants infected in utero can have many serious and permanent birth defects, including vision problems, mental retardation, and hydrocephaly (water on the brain).\",\n          \"The five main kinds of botulism are: Foodborne botulism, Wound botulism, Infant botulism, Adult intestinal toxemia botulism, and Iatrogenic botulism. In the United States, approximately 15% of botulism cases are foodborne, 65% are infant botulism, and 20% are wound botulism. Most wound botulism cases are associated with black-tar heroin injection, especially in California.\"\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"reference_context\",\n      \"properties\": {\n        \"dtype\": \"string\",\n        \"num_unique_values\": 24,\n        \"samples\": [\n          \"Document 84: Botulism can result in death due to respiratory failure. However, in the past 50 years the proportion of patients with botulism who die has fallen from about 50% to 3-5%. A patient with severe botulism may require a breathing machine as well as intensive medical and nursing care for several months, and some patients die from infections or other problems related to remaining paralyzed for weeks or months. Patients who survive an episode of botulism poisoning may have fatigue and shortness of breath for years and long-term therapy may be needed to aid recovery.\\n\\nDocument 83: The respiratory failure and paralysis that occur with severe botulism may require a patient to be on a breathing machine (ventilator) for weeks or months, plus intensive medical and nursing care. The paralysis slowly improves. Botulism can be treated with an antitoxin which blocks the action of toxin circulating in the blood. Antitoxin for infants is available from the California Department of Public Health, and antitoxin for older children and adults is available through CDC.If given before paralysis is complete, antitoxin can prevent worsening and shorten recovery time. Physicians may try to remove contaminated food still in the gut by inducing vomiting or by using enemas. Wounds should be treated, usually surgically, to remove the source of the toxin-producing bacteria followed by administration of appropriate antibiotics. Good supportive care in a hospital is the mainstay of therapy for all forms of botulism.\\n\\nDocument 81: The classic symptoms of botulism include double vision, blurred vision, drooping eyelids, slurred speech, difficulty swallowing, dry mouth, and muscle weakness. Infants with botulism appear lethargic, feed poorly, are constipated, and have a weak cry and poor muscle tone. These are all symptoms of the muscle paralysis caused by the bacterial toxin. If untreated, these symptoms may progress to cause paralysis of the respiratory muscles, arms, legs, and trunk. In foodborne botulism, symptoms generally begin 18 to 36 hours after eating a contaminated food, but they can occur as early as 6 hours or as late as 10 days.\",\n          \"Document 55: Transmission of AHFV is not well understood. AHFV is a zoonotic virus, and its described tick hosts (the soft tick Ornithodoros savignyi and the hard tick Hyalomma dromedari) are widely distributed. People can become infected through a tick bite or when crushing infected ticks. Epidemiologic studies indicate that contact with domestic animals or livestock may increase the risk of human infection. No human-to-human transmission of AHF has been documented. \\n \\nAlthough livestock animals may provide blood meals for ticks, it is thought that they play a minor role in transmitting AHFV to humans. No transmission through non-pasteurized milk has been described, although other tick-borne flaviviruses have been transmitted to humans through this route.\\n\\nDocument 60: Given that no treatment or specific prophylaxis is presently available, prevention and increased awareness of AHFV are the only recommended measures. Complete control of ticks and interruption of the virus life cycle is impractical; in endemic regions, it is important to avoid tick-infested areas and to limit contact with livestock and domestic animals.  \\n                \\nIndividuals should use tick repellants on skin and clothes and check skin for attached ticks, removing them as soon as possible. Tick collars are available for domestic animals, and dipping in acaricides is effective in killing ticks on livestock. People working with animals or animal products in farms or slaughterhouses should avoid unprotected contact with the blood, fluids, or tissues of any potentially infected or viremic animals.\",\n          \"Document 70: Baylisascaris infection can be prevented by avoiding contact with raccoons and their feces. Washing your hands after working or playing outdoors is good practice for preventing a number of diseases.\\n    \\nDo not keep, feed, or adopt wild animals, including raccoons, as pets. Infection rarely causes symptoms in raccoons, so you cannot tell if a raccoon is infected by observing its behavior. Roundworm eggs passed in the feces of infected raccoons are not visible to the naked eye. Eggs can only be seen using a microscope.\\n    \\n     \\n    \\n    \\nYou may discourage raccoons from living in and around your home or parks by taking these steps:\\n    \\n     - prevent access to food\\n     - keep trash containers tightly closed\\n     - close off access to attics and basements\\n     - keep sandboxes covered when not in use (raccoons may use sandboxes as a latrine)\\n     - remove fish ponds -- they eat the fish and drink the water\\n     - eliminate water sources\\n     - remove bird feeders\\n     - clear brush so raccoons are not likely to make a den on your property\\n    \\n    \\nStay away from areas and materials that might be contaminated by raccoon feces. Raccoons typically defecate at the base of or in raised forks of trees, or on raised horizontal surfaces such as fallen logs, stumps, or large rocks. Raccoon feces also can be found on woodpiles, decks, rooftops, and in attics, garages, and haylofts. Feces usually are dark and tubular, have a pungent odor (usually worse than dog or cat feces), and often contain undigested seeds or other food items.\\n    \\nIf you have found a raccoon latrine near your home, cleaning the area may prevent possible infection. Newly deposited eggs take at least 2-4 weeks to become infective. Prompt removal and destruction of raccoon feces will reduce risk for exposure and possible infection.\\n    \\nMore on: Raccoon Latrine Clean-up [PDF, 111 KB, 1 page]\\n    \\n     \\n    \\n    \\nIf you choose to clean the site yourself, care should be taken to avoid contaminating hands and clothes.\\n    \\n     - Wear disposable gloves to help prevent cross contamination.\\n     - Wear a N95-rated respirator if working in a confined space to prevent accidental ingestion of eggs or other harmful materials.\\n     - Avoid stirring up dust and debris- you can lightly mist the latrine area with a little water from a spray bottle to reduce the amount of dust.\\n     - Wear rubber boots that can be scrubbed or cover your shoes with disposable booties that can be thrown away, so that you do not bring eggs into your household.\\n     - Feces and material contaminated with raccoon feces should be removed and burned, buried, or sent to a landfill.\\n     - Most chemicals do not kill roundworm eggs; however, heat kills the eggs instantly.\\n     - Treat feces-soiled decks, patios, and other surfaces with boiling water or a propane torch (please contact your local fire department for regulations and safety practices).\\n    \\n    \\nTo help further reduce the risk of possible infection, wash your hands well with soap and warm running water. Clean/launder your clothes thoroughly with hot water and detergent.\\n    \\nMore on: Handwashing\\n    \\nIf you are cleaning an indoor raccoon latrine and are not able to use a propane torch, use a damp (but not wet) sponge to wipe the area with hot soapy water. Rinse your sponge frequently. After you are finished, flush dirty water down the toilet. Place the sponge in a plastic bag and put the plastic bag in the garbage.\\n    \\nContact your local animal control office for additional assistance.\\n    Dogs\\n    \\nDogs may be infected with adult B. procyonis roundworms, but may not show symptoms. Have all pets de-wormed under a veterinarian's supervision and take precautions to avoid contact with their feces.\\n    Exotic pets\\n    \\nRaccoons and dogs are not the only hosts of Baylisascaris. B. procyonis infection has also been documented in kinkajous. Other animals such as coatis may be susceptible. When wild animals are kept as pets, there can be a risk of disease transmission to humans.\\n\\nDocument 67: Raccoons are the primary, or definitive, host of Baylisascaris procyonis, a roundworm. Raccoons become infected with Baylisascaris in one of two ways:\\n    \\n     - Young raccoons become infected by eating eggs during foraging, feeding, and grooming.\\n     - Adult raccoons acquire the infection by eating rodents, rabbits, and birds infected with the larvae of Baylisascaris.\\n    \\n    \\nInfected raccoons have been found throughout the United States, mainly in the Midwest, Northeast, Middle Atlantic, and West Coast.\\n    \\nRaccoons are peridomestic animals, which means they live in or around areas where people live. Roundworm eggs are passed in the feces of infected raccoons. Raccoons defecate in communal sites, called latrines. Raccoon latrines are often found at bases of trees, unsealed attics, or on flat surfaces such as logs, tree stumps, rocks, decks, and rooftops. As more raccoons move into populated areas, the number and density of their latrines will increase.\\n    \\nWhile raccoons are the roundworm's primary host, other types of animals can become infected. Birds and small mammals, such as rodents and rabbits, are susceptible to the parasite. Unlike raccoons, these animals sometimes show signs of infection, such as muscle spasms, tremors, and progressive weakness; infection can lead to death. Predator animals, including dogs, may become infected by eating an animal that has been infected with Baylisascaris. In some dogs, Baylisascaris may develop to adult worms and pass eggs in the dogs' feces.\\n    \\nThe worms develop to maturity in the raccoon intestine, where they produce millions of eggs that are passed in the feces. Eggs that are excreted by raccoons are not immediately infectious. These eggs must develop in the environment for 2 to 4 weeks, after which the eggs are able to cause infection. The eggs are resistant to most environmental conditions and with adequate moisture, can survive for years.\\n    \\n     \\n    \\n    \\nHumans become infected by ingesting embryonated (fertile) eggs. Anyone who is exposed to environments where raccoons frequent is potentially at risk. Young children or developmentally disabled persons are at highest risk for infection as they may be more likely to put contaminated fingers, soil, or objects into their mouths.\\n    \\nHunters, trappers, taxidermists, and wildlife handlers may also be at increased risk if they have contact with raccoons or raccoon habitats.\\n    \\nFewer than 25 cases of Baylisascaris disease have been documented in the United States. However, it is possible that some cases are incorrectly diagnosed as other infections or go undiagnosed. Cases that are diagnosed tend to be severe.\\n    \\nCases have been reported in California, Illinois, Louisiana, Massachusetts, Michigan, Minnesota, Missouri, New York, Oregon, and Pennsylvania. As of 2012, there were 16 published human neurological cases in the US; six of the infected persons died.\\n\\nDocument 66: Baylisascaris worms are intestinal parasites found in a wide variety of animals. Different species of Baylisascaris are associated with different animal hosts. For example, Baylisascaris procyonis is found in raccoons and Baylisascaris columnaris is an intestinal parasite found in skunks. Cases of Baylisascaris infection in people are not frequently reported, but can be severe. Baylisascaris procyonis is thought to pose the greatest risk to humans because of the often close association of raccoons to human dwellings.\"\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"conversation_history\",\n      \"properties\": {\n        \"dtype\": \"object\",\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"metadata\",\n      \"properties\": {\n        \"dtype\": \"object\",\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    }\n  ]\n}"
            }
          },
          "metadata": {},
          "execution_count": 133
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "A function is created that will be called from Giskard's evaluate function. This function receives the question and returns the agent's response."
      ],
      "metadata": {
        "id": "21tFRKfXC2wF"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "def use_agent(question, history=None):\n",
        "    return agent_executor.invoke({\"input\": question})"
      ],
      "metadata": {
        "id": "InRbIK2_V2HU"
      },
      "execution_count": 168,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "report = evaluate(use_agent, testset=test_questions, knowledge_base=kb_giskard)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000,
          "referenced_widgets": [
            "5a72094bcf8849478e7f67534c82b986",
            "a8e8bf71c3ec48d48651815da43ed87b",
            "24465a52647144f68e1790097be7322f",
            "986027d02f924baa8ab8577b784c0ae0",
            "fc943a9d4af04f2d8fa3fb84f28e93c9",
            "79136cd9901f4c9897ab4aece1ed4dd9",
            "24dd887e6a6e463f83dda16487b3a3c4",
            "90ad8de1a70840acb49adc1ade25987f",
            "e0ebf112d8a5438f8011c58a67b46e91",
            "777dd928e66343bb853d84ce41fee52d",
            "cf3686a938fc4d5e837b508c05105252",
            "0467453ef02b44b184966eac732df1f8",
            "75ede69d28f34af2874d18b8027acb2c",
            "f71d882f15934e7ea939cdd7c2352c98",
            "effa5f9248464a649bc9f63c46c9aa1d",
            "f7424a8cd83940fe82e1be8813a7df35",
            "c941895af5614ff087907ec478808184",
            "0ec6f8c9dffe41d7aff4be8ee208556d",
            "30a0be6e8f5240c0b852d2633dd5827e",
            "d73162b994cd45168c10197d8d1bfeb8",
            "c3b15d415d254e10a9c79de92cc043c1",
            "f74fd76b49344088829f405db7e82e68"
          ]
        },
        "id": "D3fm-_7RU4y9",
        "outputId": "8086e9e6-1a79-4d95-95f7-44c5763d031c"
      },
      "execution_count": 169,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "Asking questions to the agent:   0%|          | 0/30 [00:00<?, ?it/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "5a72094bcf8849478e7f67534c82b986"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "\n",
            "\u001b[1m> Entering new AgentExecutor chain...\u001b[0m\n",
            "\u001b[32;1m\u001b[1;3m\n",
            "Thought: Do I need to use a tool? Yes\n",
            "Action: Medical KB\n",
            "Action Input: Baylisascaris\u001b[0m\u001b[36;1m\u001b[1;3m Baylisascaris is a genus of intestinal parasites that can be found in various animals, with different species being associated with different animal hosts. It can cause severe infections in humans, with Baylisascaris procyonis, found in raccoons, posing the greatest risk due to their close association with human dwellings.\u001b[0m\u001b[32;1m\u001b[1;3m Do I need to use a tool? No\n",
            "Final Answer: Humans can become infected with Baylisascaris through ingestion of contaminated soil, water, or food, or through direct contact with infected animals. It is important to practice good hygiene and avoid contact with potentially contaminated sources to prevent infection.\u001b[0m\n",
            "\n",
            "\u001b[1m> Finished chain.\u001b[0m\n",
            "\n",
            "\n",
            "\u001b[1m> Entering new AgentExecutor chain...\u001b[0m\n",
            "\u001b[32;1m\u001b[1;3m\n",
            "Thought: Do I need to use a tool? Yes\n",
            "Action: Medical KB\n",
            "Action Input: Preventing and controlling the spread of body lice\u001b[0m\u001b[36;1m\u001b[1;3m The steps that can be taken to help prevent and control the spread of body lice include regularly bathing and changing into properly laundered clothes, washing and drying infested clothing and bedding with hot water and high heat, avoiding sharing clothing and bedding with an infested person, and potentially using fumigation or chemical insecticides in certain cases.\u001b[0m\u001b[32;1m\u001b[1;3m Do I need to use a tool? No\n",
            "Final Answer: Some steps that can help prevent and control the spread of body lice include regularly bathing and changing into properly laundered clothes, washing and drying infested clothing and bedding with hot water and high heat, avoiding sharing clothing and bedding with an infested person, and potentially using fumigation or chemical insecticides in certain cases. It is also important to seek medical treatment if you suspect an infestation.\u001b[0m\n",
            "\n",
            "\u001b[1m> Finished chain.\u001b[0m\n",
            "\n",
            "\n",
            "\u001b[1m> Entering new AgentExecutor chain...\u001b[0m\n",
            "\u001b[32;1m\u001b[1;3m\n",
            "Thought: Do I need to use a tool? Yes\n",
            "Action: Medical KB\n",
            "Action Input: Scabies\u001b[0m\u001b[36;1m\u001b[1;3m Scabies is an infestation of the skin caused by the human itch mite. It is spread through direct, prolonged skin-to-skin contact and is commonly found in crowded living conditions and institutions such as nursing homes and prisons. Symptoms include intense itching and a pimple-like rash.\u001b[0m\u001b[32;1m\u001b[1;3mDo I need to use a tool? No\n",
            "Final Answer: The recommended prevention method for scabies is to avoid direct, prolonged skin-to-skin contact with infected individuals and to maintain good hygiene practices. It is also important to seek medical treatment if you suspect an infestation.\u001b[0m\n",
            "\n",
            "\u001b[1m> Finished chain.\u001b[0m\n",
            "\n",
            "\n",
            "\u001b[1m> Entering new AgentExecutor chain...\u001b[0m\n",
            "\u001b[32;1m\u001b[1;3m\n",
            "Thought: Do I need to use a tool? Yes\n",
            "Action: Medical KB\n",
            "Action Input: Botulism\u001b[0m\u001b[36;1m\u001b[1;3m Botulism is a rare but serious paralytic illness caused by a nerve toxin produced by certain bacteria. There are five main types of botulism, including foodborne, wound, infant, adult intestinal toxemia, and iatrogenic. All forms of botulism can be fatal and are considered medical emergencies. Foodborne botulism is a public health emergency because it can affect many people if they consume contaminated food.\u001b[0m\u001b[32;1m\u001b[1;3mDo I need to use a tool? Yes\n",
            "Action: Medical KB\n",
            "Action Input: Botulism treatment\u001b[0m\u001b[36;1m\u001b[1;3m Botulism can be treated with an antitoxin, which blocks the action of the toxin circulating in the blood. Antitoxin is available for infants from the California Department of Public Health and for older children and adults through the CDC. Physicians may also try to remove contaminated food from the gut and treat wounds to remove the source of the toxin-producing bacteria. Good supportive care in a hospital is also important for recovery.\u001b[0m\u001b[32;1m\u001b[1;3mDo I need to use a tool? No\n",
            "Final Answer: Treatment for botulism typically involves the use of an antitoxin and supportive care in a hospital setting. It is important to seek medical treatment immediately if you suspect you or someone you know may have botulism.\u001b[0m\n",
            "\n",
            "\u001b[1m> Finished chain.\u001b[0m\n",
            "\n",
            "\n",
            "\u001b[1m> Entering new AgentExecutor chain...\u001b[0m\n",
            "\u001b[32;1m\u001b[1;3m\n",
            "Thought: Do I need to use a tool? Yes\n",
            "Action: Medical KB\n",
            "Action Input: Rabies\u001b[0m\u001b[36;1m\u001b[1;3m Each state collects specific information about rabies, and is the best source for information on rabies in your area. In addition, the CDC publishes rabies surveillance data every year for the United States. The report, entitled Rabies Surveillance in the United States, contains information about the number of cases of rabies reported to CDC during the year, the animals reported rabid, maps showing where cases were reported for wild and domestic animals, and distribution maps showing outbreaks of rabies associated with specific animals.\u001b[0m\u001b[32;1m\u001b[1;3m Do I need to use a tool? No\n",
            "Final Answer: The first symptoms of rabies in humans include fever, headache, and general weakness or discomfort. As the disease progresses, symptoms may include insomnia, anxiety, confusion, paralysis, and hallucinations. To prevent rabies, it is important to avoid contact with wild animals and to vaccinate pets against the disease. If you are bitten by an animal, seek medical attention immediately.\u001b[0m\n",
            "\n",
            "\u001b[1m> Finished chain.\u001b[0m\n",
            "\n",
            "\n",
            "\u001b[1m> Entering new AgentExecutor chain...\u001b[0m\n",
            "\u001b[32;1m\u001b[1;3m\n",
            "Thought: Do I need to use a tool? Yes\n",
            "Action: Medical KB\n",
            "Action Input: Lymphocytic choriomeningitis virus (LCMV)\u001b[0m\u001b[36;1m\u001b[1;3m LCMV is a virus that can cause neurological disease, meningitis, encephalitis, and other symptoms such as fever, muscle aches, and headache. It can also cause birth defects in infants if a pregnant woman becomes infected. LCMV is usually not fatal, but can cause temporary or permanent neurological damage.\u001b[0m\u001b[32;1m\u001b[1;3mDo I need to use a tool? No\n",
            "Final Answer: LCMV can be transmitted through contact with infected rodents, their droppings, or contaminated food or water. It can also be transmitted from mother to child during pregnancy or through organ transplantation. People who work with rodents or live in close proximity to them, such as in rural areas, may be at a higher risk of contracting LCMV. It is important to take precautions to avoid contact with rodents and to seek medical attention if you suspect you have been exposed to LCMV.\u001b[0m\n",
            "\n",
            "\u001b[1m> Finished chain.\u001b[0m\n",
            "\n",
            "\n",
            "\u001b[1m> Entering new AgentExecutor chain...\u001b[0m\n",
            "\u001b[32;1m\u001b[1;3m\n",
            "Thought: Do I need to use a tool? Yes\n",
            "Action: Medical KB\n",
            "Action Input: Seafood, naturally occurring marine toxins\u001b[0m\u001b[36;1m\u001b[1;3m The seafood can be contaminated with marine toxins, which can cause disease when consumed by humans.\u001b[0m\u001b[32;1m\u001b[1;3m Do I need to use a tool? No\n",
            "Final Answer: Seafood that has been tainted with naturally occurring marine toxins may have a distinct appearance, such as a slimy texture or unusual color. It may also have a strong odor or taste. Consuming this seafood can lead to symptoms such as nausea, vomiting, and diarrhea. It is important to avoid consuming seafood that may be contaminated with marine toxins and to seek medical attention if you experience any symptoms after consuming seafood.\u001b[0m\n",
            "\n",
            "\u001b[1m> Finished chain.\u001b[0m\n",
            "\n",
            "\n",
            "\u001b[1m> Entering new AgentExecutor chain...\u001b[0m\n",
            "\u001b[32;1m\u001b[1;3m\n",
            "Thought: Do I need to use a tool? Yes\n",
            "Action: Medical KB\n",
            "Action Input: Acanthamoeba keratitis and Granulomatous Amebic Encephalitis (GAE)\u001b[0m\u001b[36;1m\u001b[1;3m Early diagnosis is essential for effective treatment of both Acanthamoeba keratitis and Granulomatous Amebic Encephalitis (GAE). While Acanthamoeba keratitis can be diagnosed by an eye specialist based on symptoms and various tests, GAE is more difficult to diagnose and may require brain scans, biopsies, or spinal taps.\u001b[0m\u001b[32;1m\u001b[1;3mDo I need to use a tool? No\n",
            "Final Answer: The most prevalent risk factor for contracting Acanthamoeba keratitis and Granulomatous Amebic Encephalitis (GAE) is exposure to contaminated water or soil, particularly for those who wear contact lenses or have weakened immune systems. It is important to take precautions when swimming or using contact lenses to reduce the risk of contracting these infections.\u001b[0m\n",
            "\n",
            "\u001b[1m> Finished chain.\u001b[0m\n",
            "\n",
            "\n",
            "\u001b[1m> Entering new AgentExecutor chain...\u001b[0m\n",
            "\u001b[32;1m\u001b[1;3m\n",
            "Thought: Do I need to use a tool? Yes\n",
            "Action: Medical KB\n",
            "Action Input: Botulism treatment\u001b[0m\u001b[36;1m\u001b[1;3m Botulism can be treated with an antitoxin, which blocks the action of the toxin circulating in the blood. Antitoxin is available for infants from the California Department of Public Health and for older children and adults through the CDC. Physicians may also try to remove contaminated food from the gut and treat wounds to remove the source of the toxin-producing bacteria. Good supportive care in a hospital is also important for recovery.\u001b[0m\u001b[32;1m\u001b[1;3m Do I need to use a tool? No\n",
            "Final Answer: Botulism can be treated with an antitoxin, which blocks the action of the toxin circulating in the blood. Antitoxin is available for infants from the California Department of Public Health and for older children and adults through the CDC. Physicians may also try to remove contaminated food from the gut and treat wounds to remove the source of the toxin-producing bacteria. Good supportive care in a hospital is also important for recovery.\u001b[0m\n",
            "\n",
            "\u001b[1m> Finished chain.\u001b[0m\n",
            "\n",
            "\n",
            "\u001b[1m> Entering new AgentExecutor chain...\u001b[0m\n",
            "\u001b[32;1m\u001b[1;3m\n",
            "Thought: Do I need to use a tool? No\n",
            "Final Answer: Toxoplasmosis can be transmitted through ingestion of contaminated food or water, contact with infected cat feces, or from mother to child during pregnancy. Symptoms may include flu-like symptoms, muscle aches, and swollen lymph nodes. It is important to take precautions to avoid exposure to the parasite and to seek medical attention if you suspect you have been infected.\u001b[0m\n",
            "\n",
            "\u001b[1m> Finished chain.\u001b[0m\n",
            "\n",
            "\n",
            "\u001b[1m> Entering new AgentExecutor chain...\u001b[0m\n",
            "\u001b[32;1m\u001b[1;3m\n",
            "Thought: Do I need to use a tool? Yes\n",
            "Action: Medical KB\n",
            "Action Input: Preventive measures for body lice and whipworm\u001b[0m\u001b[36;1m\u001b[1;3m The best way to prevent whipworm infection is to avoid ingesting contaminated soil and to practice good hand hygiene. Transmission of infection to others can also be prevented by not defecating outdoors and having effective sewage disposal systems.\u001b[0m\u001b[32;1m\u001b[1;3mDo I need to use a tool? No\n",
            "Final Answer: Some preventive measures to control the spread of body lice include regularly washing and changing clothes and bedding, avoiding sharing personal items, and treating infested individuals with medicated shampoos or lotions. It is also important to maintain good personal hygiene and to seek medical treatment if necessary.\u001b[0m\n",
            "\n",
            "\u001b[1m> Finished chain.\u001b[0m\n",
            "\n",
            "\n",
            "\u001b[1m> Entering new AgentExecutor chain...\u001b[0m\n",
            "\u001b[32;1m\u001b[1;3m\n",
            "Thought: Do I need to use a tool? Yes\n",
            "Action: Medical KB\n",
            "Action Input: Taenia tapeworm infection\u001b[0m\u001b[36;1m\u001b[1;3m Taenia tapeworm infection is a parasitic infection caused by the tapeworm species Taenia saginata, Taenia solium, and Taenia asiatica. It can be contracted by consuming raw or undercooked beef or pork. Symptoms are usually mild or nonexistent, but T. solium infections can lead to a more serious condition called cysticercosis, which can cause seizures. Treatment is important for T. solium infections to prevent complications.\u001b[0m\u001b[32;1m\u001b[1;3m Do I need to use a tool? No\n",
            "Final Answer: Various laboratory methods can be used to diagnose a Taenia tapeworm infection, including stool sample analysis, blood tests, and imaging tests such as ultrasound or MRI. These methods can help identify the presence of tapeworm eggs or larvae in the body.\u001b[0m\n",
            "\n",
            "\u001b[1m> Finished chain.\u001b[0m\n",
            "\n",
            "\n",
            "\u001b[1m> Entering new AgentExecutor chain...\u001b[0m\n",
            "\u001b[32;1m\u001b[1;3m\n",
            "Thought: Do I need to use a tool? No\n",
            "Final Answer: Good hygiene practices, such as regularly washing and changing clothes and bedding, avoiding sharing personal items, and seeking medical treatment if necessary, can help prevent the spread of head lice and potentially exacerbating the condition of botulism.\u001b[0m\n",
            "\n",
            "\u001b[1m> Finished chain.\u001b[0m\n",
            "\n",
            "\n",
            "\u001b[1m> Entering new AgentExecutor chain...\u001b[0m\n",
            "\u001b[32;1m\u001b[1;3m\n",
            "Thought: Do I need to use a tool? Yes\n",
            "Action: Medical KB\n",
            "Action Input: cysticercosis\u001b[0m\u001b[36;1m\u001b[1;3m Cysticercosis is an infection caused by the larvae of the parasite Taenia solium, which occurs when a person swallows tapeworm eggs. The larvae can form cysts in tissues such as muscle and brain, leading to a condition called neurocysticercosis when cysts are found in the brain.\u001b[0m\u001b[32;1m\u001b[1;3m Do I need to use a tool? No\n",
            "Final Answer: To prevent and control the spread of body lice, it is important to maintain good personal hygiene, regularly wash and change clothes and bedding, avoid sharing personal items, and seek medical treatment if necessary. Additionally, it is important to take precautions to avoid exposure to the parasite and to seek medical attention if you suspect you have been infected.\u001b[0m\n",
            "\n",
            "\u001b[1m> Finished chain.\u001b[0m\n",
            "\n",
            "\n",
            "\u001b[1m> Entering new AgentExecutor chain...\u001b[0m\n",
            "\u001b[32;1m\u001b[1;3mThought: Do I need to use a tool? Yes\n",
            "Action: Medical KB\n",
            "Action Input: LCMV infection during pregnancy\u001b[0m\u001b[36;1m\u001b[1;3m LCMV infection can be transmitted from an infected mother to her fetus during pregnancy.\u001b[0m\u001b[32;1m\u001b[1;3mDo I need to use a tool? No\n",
            "Final Answer: If a LCMV infection during pregnancy goes unreported or undetected, it could potentially lead to serious health complications for both the mother and the fetus. It is important to seek medical attention and report any potential infections to prevent further spread and potential harm.\u001b[0m\n",
            "\n",
            "\u001b[1m> Finished chain.\u001b[0m\n",
            "\n",
            "\n",
            "\u001b[1m> Entering new AgentExecutor chain...\u001b[0m\n",
            "\u001b[32;1m\u001b[1;3m\n",
            "Thought: Do I need to use a tool? No\n",
            "Final Answer: If your son has contracted LCMV during your pregnancy, it could potentially lead to serious health complications for both him and the fetus. It is important to seek medical attention and report any potential infections to prevent further spread and potential harm.\u001b[0m\n",
            "\n",
            "\u001b[1m> Finished chain.\u001b[0m\n",
            "\n",
            "\n",
            "\u001b[1m> Entering new AgentExecutor chain...\u001b[0m\n",
            "\u001b[32;1m\u001b[1;3m\n",
            "Thought: Do I need to use a tool? Yes\n",
            "Action: Medical KB\n",
            "Action Input: Babesiosis\u001b[0m\u001b[36;1m\u001b[1;3m Babesiosis is a tickborne infection caused by microscopic parasites that infect red blood cells. It is most commonly caused by the parasite Babesia microti and is spread by Ixodes scapularis ticks. The infection can range in severity and is both treatable and preventable.\u001b[0m\u001b[32;1m\u001b[1;3mDo I need to use a tool? No\n",
            "Final Answer: Babesiosis is primarily spread through tick bites, so it is important to take precautions such as wearing protective clothing and using insect repellent when hiking in areas where the parasite is prevalent. It is also important to check for ticks after spending time outdoors and to seek medical attention if you suspect you have been bitten by a tick.\u001b[0m\n",
            "\n",
            "\u001b[1m> Finished chain.\u001b[0m\n",
            "\n",
            "\n",
            "\u001b[1m> Entering new AgentExecutor chain...\u001b[0m\n",
            "\u001b[32;1m\u001b[1;3mThought: Do I need to use a tool? Yes\n",
            "Action: Medical KB\n",
            "Action Input: whipworm infection\u001b[0m\u001b[36;1m\u001b[1;3m Whipworm infection is caused by the parasitic roundworm, whipworm, and is most commonly found in areas with tropical weather and poor sanitation practices. It is spread through fecal-oral transmission and can occur in both children and adults. In 2002, it was estimated that 1 billion people were infected with whipworm worldwide. Trichuriasis, the infection caused by whipworm, also occurs in the southern United States.\u001b[0m\u001b[32;1m\u001b[1;3m Do I need to use a tool? No\n",
            "Final Answer: To prevent whipworm infection, it is important to practice good hygiene, including washing hands before eating and after using the bathroom, avoiding contaminated food and water, and properly disposing of feces. Additionally, avoiding contact with soil that may be contaminated with whipworm eggs can also help prevent infection. If you suspect your child may have contracted a whipworm infection, it is important to seek medical attention for proper diagnosis and treatment.\u001b[0m\n",
            "\n",
            "\u001b[1m> Finished chain.\u001b[0m\n",
            "\n",
            "\n",
            "\u001b[1m> Entering new AgentExecutor chain...\u001b[0m\n",
            "\u001b[32;1m\u001b[1;3m\n",
            "Thought: Do I need to use a tool? Yes\n",
            "Action: Medical KB\n",
            "Action Input: AHFV prevention\u001b[0m\u001b[36;1m\u001b[1;3m The only recommended measures for AHFV prevention are prevention and increased awareness.\u001b[0m\u001b[32;1m\u001b[1;3m Do I need to use a tool? No\n",
            "Final Answer: To prevent the spread of AHFV among your livestock and yourself, it is important to practice good hygiene, such as washing your hands before and after handling animals, and to avoid contact with potentially infected animals or their bodily fluids. Additionally, it is important to monitor your livestock for any signs of illness and to seek medical attention if you suspect you or your animals may have been exposed to AHFV.\u001b[0m\n",
            "\n",
            "\u001b[1m> Finished chain.\u001b[0m\n",
            "\n",
            "\n",
            "\u001b[1m> Entering new AgentExecutor chain...\u001b[0m\n",
            "\u001b[32;1m\u001b[1;3m\n",
            "Thought: Do I need to use a tool? No\n",
            "Final Answer: To prevent contracting the Marburg virus, it is important to avoid contact with infected individuals or their bodily fluids, as well as avoiding contact with potentially infected animals. It is also important to practice good hygiene, such as washing your hands frequently and avoiding consuming raw or undercooked meat. Additionally, it is recommended to avoid traveling to areas with known outbreaks of the virus.\u001b[0m\n",
            "\n",
            "\u001b[1m> Finished chain.\u001b[0m\n",
            "\n",
            "\n",
            "\u001b[1m> Entering new AgentExecutor chain...\u001b[0m\n",
            "\u001b[32;1m\u001b[1;3m\n",
            "Thought: Do I need to use a tool? Yes\n",
            "Action: Medical KB\n",
            "Action Input: Rabies\u001b[0m\u001b[36;1m\u001b[1;3m Each state collects specific information about rabies, and is the best source for information on rabies in your area. In addition, the CDC publishes rabies surveillance data every year for the United States. The report, entitled Rabies Surveillance in the United States, contains information about the number of cases of rabies reported to CDC during the year, the animals reported rabid, maps showing where cases were reported for wild and domestic animals, and distribution maps showing outbreaks of rabies associated with specific animals.\u001b[0m\u001b[32;1m\u001b[1;3mDo I need to use a tool? No\n",
            "Final Answer: The initial symptoms of rabies can include fever, headache, and weakness. As the disease progresses, symptoms can include anxiety, confusion, and agitation. Once symptoms appear, the disease typically progresses rapidly and can lead to coma and death. It is important to seek medical attention immediately if you suspect you may have been exposed to rabies.\u001b[0m\n",
            "\n",
            "\u001b[1m> Finished chain.\u001b[0m\n",
            "\n",
            "\n",
            "\u001b[1m> Entering new AgentExecutor chain...\u001b[0m\n",
            "\u001b[32;1m\u001b[1;3m\n",
            "Thought: Do I need to use a tool? Yes\n",
            "Action: Medical KB\n",
            "Action Input: AHFV transmission\u001b[0m\u001b[36;1m\u001b[1;3m The transmission of AHFV is not well understood, but it is thought to primarily occur through tick bites or crushing infected ticks. Contact with domestic animals or livestock may also increase the risk of human infection, but it is not a major mode of transmission. There is no evidence of human-to-human transmission of AHFV, and it has not been transmitted through non-pasteurized milk.\u001b[0m\u001b[32;1m\u001b[1;3m Do I need to use a tool? Yes\n",
            "Action: Medical KB\n",
            "Action Input: AHFV prevention\u001b[0m\u001b[36;1m\u001b[1;3m The only recommended measures for AHFV prevention are prevention and increased awareness.\u001b[0m\u001b[32;1m\u001b[1;3m Do I need to use a tool? No\n",
            "Final Answer: To prevent the spread of AHFV, it is important to practice good hygiene, such as washing your hands before and after handling animals, and to avoid contact with potentially infected animals or their bodily fluids. Additionally, it is important to monitor your livestock for any signs of illness and to seek medical attention if you suspect you or your animals may have been exposed to AHFV.\u001b[0m\n",
            "\n",
            "\u001b[1m> Finished chain.\u001b[0m\n",
            "\n",
            "\n",
            "\u001b[1m> Entering new AgentExecutor chain...\u001b[0m\n",
            "\u001b[32;1m\u001b[1;3m\n",
            "Thought: Do I need to use a tool? Yes\n",
            "Action: Medical KB\n",
            "Action Input: Botulism\u001b[0m\u001b[36;1m\u001b[1;3m Botulism is a rare but serious paralytic illness caused by a nerve toxin produced by certain bacteria. There are five main types of botulism, including foodborne, wound, infant, adult intestinal toxemia, and iatrogenic. All forms of botulism can be fatal and are considered medical emergencies. Foodborne botulism is a public health emergency because it can affect many people if they consume contaminated food.\u001b[0m\u001b[32;1m\u001b[1;3mDo I need to use a tool? No\n",
            "Final Answer: To diagnose botulism, a doctor may perform a physical exam, review symptoms and medical history, and order laboratory tests, such as a stool or blood test. Other diseases that may appear similar to botulism include Guillain-Barré syndrome, myasthenia gravis, and stroke.\u001b[0m\n",
            "\n",
            "\u001b[1m> Finished chain.\u001b[0m\n",
            "\n",
            "\n",
            "\u001b[1m> Entering new AgentExecutor chain...\u001b[0m\n",
            "\u001b[32;1m\u001b[1;3m\n",
            "Thought: Do I need to use a tool? Yes\n",
            "Action: Medical KB\n",
            "Action Input: Botulism\u001b[0m\u001b[36;1m\u001b[1;3m Botulism is a rare but serious paralytic illness caused by a nerve toxin produced by certain bacteria. There are five main types of botulism, including foodborne, wound, infant, adult intestinal toxemia, and iatrogenic. All forms of botulism can be fatal and are considered medical emergencies. Foodborne botulism is a public health emergency because it can affect many people if they consume contaminated food.\u001b[0m\u001b[32;1m\u001b[1;3mDo I need to use a tool? No\n",
            "Final Answer: The five main types of botulism are foodborne, wound, infant, adult intestinal toxemia, and iatrogenic. Each type is caused by different factors, such as consuming contaminated food or having a wound infected with the bacteria that produce the botulinum toxin.\u001b[0m\n",
            "\n",
            "\u001b[1m> Finished chain.\u001b[0m\n",
            "\n",
            "\n",
            "\u001b[1m> Entering new AgentExecutor chain...\u001b[0m\n",
            "\u001b[32;1m\u001b[1;3m\n",
            "Thought: Do I need to use a tool? Yes\n",
            "Action: Medical KB\n",
            "Action Input: Baylisascaris worms\u001b[0m\u001b[36;1m\u001b[1;3m Baylisascaris worms are intestinal parasites found in a wide variety of animals.\u001b[0m\u001b[32;1m\u001b[1;3mDo I need to use a tool? No\n",
            "Final Answer: Humans can become infected by Baylisascaris worms by accidentally ingesting the eggs of the parasite, which can be found in soil, water, or on surfaces contaminated with animal feces. To prevent this infection, it is important to practice good hygiene, such as washing your hands after handling animals or potentially contaminated objects, and to avoid contact with animal feces. Additionally, it is important to properly dispose of animal waste and to regularly deworm pets and livestock.\u001b[0m\n",
            "\n",
            "\u001b[1m> Finished chain.\u001b[0m\n",
            "\n",
            "\n",
            "\u001b[1m> Entering new AgentExecutor chain...\u001b[0m\n",
            "\u001b[32;1m\u001b[1;3mThought: Do I need to use a tool? No\n",
            "Final Answer: The five main types of botulism are foodborne, wound, infant, adult intestinal toxemia, and iatrogenic.\u001b[0m\n",
            "\n",
            "\u001b[1m> Finished chain.\u001b[0m\n",
            "\n",
            "\n",
            "\u001b[1m> Entering new AgentExecutor chain...\u001b[0m\n",
            "\u001b[32;1m\u001b[1;3mThought: Do I need to use a tool? No\n",
            "Final Answer: Achieving a goal or task can be achieved through careful planning, hard work, and determination. It is important to set specific and achievable goals, break them down into smaller steps, and stay motivated and focused on the end result. Additionally, seeking guidance and support from others can also help in achieving a goal.\u001b[0m\n",
            "\n",
            "\u001b[1m> Finished chain.\u001b[0m\n",
            "\n",
            "\n",
            "\u001b[1m> Entering new AgentExecutor chain...\u001b[0m\n",
            "\u001b[32;1m\u001b[1;3mThought: Do I need to use a tool? No\n",
            "Final Answer: There are several preventive measures that can be taken to reduce the risk of various infections and illnesses. These include practicing good hygiene, such as washing your hands regularly, avoiding contact with contaminated objects, and properly disposing of waste. It is also important to maintain a healthy lifestyle, including a balanced diet and regular exercise, to boost your immune system and prevent illnesses. Additionally, staying up to date on vaccinations and seeking medical advice when necessary can also help prevent the spread of diseases.\u001b[0m\n",
            "\n",
            "\u001b[1m> Finished chain.\u001b[0m\n",
            "\n",
            "\n",
            "\u001b[1m> Entering new AgentExecutor chain...\u001b[0m\n",
            "\u001b[32;1m\u001b[1;3mThought: Do I need to use a tool? No\n",
            "Final Answer: Some precautions that can help prevent the spread of infections and illnesses include practicing good hygiene, maintaining a healthy lifestyle, and staying up to date on vaccinations. It is also important to seek medical advice when necessary and to properly dispose of waste to avoid contamination.\u001b[0m\n",
            "\n",
            "\u001b[1m> Finished chain.\u001b[0m\n",
            "\n",
            "\n",
            "\u001b[1m> Entering new AgentExecutor chain...\u001b[0m\n",
            "\u001b[32;1m\u001b[1;3m\n",
            "Thought: Do I need to use a tool? No\n",
            "Final Answer: The five main types of botulism are foodborne, wound, infant, adult intestinal toxemia, and iatrogenic.\u001b[0m\n",
            "\n",
            "\u001b[1m> Finished chain.\u001b[0m\n"
          ]
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "Correctness evaluation:   0%|          | 0/30 [00:00<?, ?it/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "0467453ef02b44b184966eac732df1f8"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "WARNING:urllib3.connectionpool:Retrying (Retry(total=3, connect=None, read=None, redirect=None, status=None)) after connection broken by 'RemoteDisconnected('Remote end closed connection without response')': /track\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# Summary with the results.\n",
        "report.correctness_by_question_type()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 269
        },
        "id": "WNLuWCB2YJwO",
        "outputId": "748954b5-40c8-4346-e07e-f0753480cd78"
      },
      "execution_count": 137,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "                     correctness\n",
              "question_type                   \n",
              "complex                      0.8\n",
              "conversational               0.4\n",
              "distracting element          1.0\n",
              "double                       1.0\n",
              "simple                       1.0\n",
              "situational                  0.6"
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-897ee5b0-8326-4e33-829f-cfc797fd7520\" class=\"colab-df-container\">\n",
              "    <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>correctness</th>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>question_type</th>\n",
              "      <th></th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>complex</th>\n",
              "      <td>0.8</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>conversational</th>\n",
              "      <td>0.4</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>distracting element</th>\n",
              "      <td>1.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>double</th>\n",
              "      <td>1.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>simple</th>\n",
              "      <td>1.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>situational</th>\n",
              "      <td>0.6</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>\n",
              "    <div class=\"colab-df-buttons\">\n",
              "\n",
              "  <div class=\"colab-df-container\">\n",
              "    <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-897ee5b0-8326-4e33-829f-cfc797fd7520')\"\n",
              "            title=\"Convert this dataframe to an interactive table.\"\n",
              "            style=\"display:none;\">\n",
              "\n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\" viewBox=\"0 -960 960 960\">\n",
              "    <path d=\"M120-120v-720h720v720H120Zm60-500h600v-160H180v160Zm220 220h160v-160H400v160Zm0 220h160v-160H400v160ZM180-400h160v-160H180v160Zm440 0h160v-160H620v160ZM180-180h160v-160H180v160Zm440 0h160v-160H620v160Z\"/>\n",
              "  </svg>\n",
              "    </button>\n",
              "\n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    .colab-df-buttons div {\n",
              "      margin-bottom: 4px;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "    <script>\n",
              "      const buttonEl =\n",
              "        document.querySelector('#df-897ee5b0-8326-4e33-829f-cfc797fd7520 button.colab-df-convert');\n",
              "      buttonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "      async function convertToInteractive(key) {\n",
              "        const element = document.querySelector('#df-897ee5b0-8326-4e33-829f-cfc797fd7520');\n",
              "        const dataTable =\n",
              "          await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                    [key], {});\n",
              "        if (!dataTable) return;\n",
              "\n",
              "        const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "          '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "          + ' to learn more about interactive tables.';\n",
              "        element.innerHTML = '';\n",
              "        dataTable['output_type'] = 'display_data';\n",
              "        await google.colab.output.renderOutput(dataTable, element);\n",
              "        const docLink = document.createElement('div');\n",
              "        docLink.innerHTML = docLinkHtml;\n",
              "        element.appendChild(docLink);\n",
              "      }\n",
              "    </script>\n",
              "  </div>\n",
              "\n",
              "\n",
              "<div id=\"df-9a2ef1c4-3bd6-4dd9-a8e6-ed339d6ac918\">\n",
              "  <button class=\"colab-df-quickchart\" onclick=\"quickchart('df-9a2ef1c4-3bd6-4dd9-a8e6-ed339d6ac918')\"\n",
              "            title=\"Suggest charts\"\n",
              "            style=\"display:none;\">\n",
              "\n",
              "<svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "     width=\"24px\">\n",
              "    <g>\n",
              "        <path d=\"M19 3H5c-1.1 0-2 .9-2 2v14c0 1.1.9 2 2 2h14c1.1 0 2-.9 2-2V5c0-1.1-.9-2-2-2zM9 17H7v-7h2v7zm4 0h-2V7h2v10zm4 0h-2v-4h2v4z\"/>\n",
              "    </g>\n",
              "</svg>\n",
              "  </button>\n",
              "\n",
              "<style>\n",
              "  .colab-df-quickchart {\n",
              "      --bg-color: #E8F0FE;\n",
              "      --fill-color: #1967D2;\n",
              "      --hover-bg-color: #E2EBFA;\n",
              "      --hover-fill-color: #174EA6;\n",
              "      --disabled-fill-color: #AAA;\n",
              "      --disabled-bg-color: #DDD;\n",
              "  }\n",
              "\n",
              "  [theme=dark] .colab-df-quickchart {\n",
              "      --bg-color: #3B4455;\n",
              "      --fill-color: #D2E3FC;\n",
              "      --hover-bg-color: #434B5C;\n",
              "      --hover-fill-color: #FFFFFF;\n",
              "      --disabled-bg-color: #3B4455;\n",
              "      --disabled-fill-color: #666;\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart {\n",
              "    background-color: var(--bg-color);\n",
              "    border: none;\n",
              "    border-radius: 50%;\n",
              "    cursor: pointer;\n",
              "    display: none;\n",
              "    fill: var(--fill-color);\n",
              "    height: 32px;\n",
              "    padding: 0;\n",
              "    width: 32px;\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart:hover {\n",
              "    background-color: var(--hover-bg-color);\n",
              "    box-shadow: 0 1px 2px rgba(60, 64, 67, 0.3), 0 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "    fill: var(--button-hover-fill-color);\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart-complete:disabled,\n",
              "  .colab-df-quickchart-complete:disabled:hover {\n",
              "    background-color: var(--disabled-bg-color);\n",
              "    fill: var(--disabled-fill-color);\n",
              "    box-shadow: none;\n",
              "  }\n",
              "\n",
              "  .colab-df-spinner {\n",
              "    border: 2px solid var(--fill-color);\n",
              "    border-color: transparent;\n",
              "    border-bottom-color: var(--fill-color);\n",
              "    animation:\n",
              "      spin 1s steps(1) infinite;\n",
              "  }\n",
              "\n",
              "  @keyframes spin {\n",
              "    0% {\n",
              "      border-color: transparent;\n",
              "      border-bottom-color: var(--fill-color);\n",
              "      border-left-color: var(--fill-color);\n",
              "    }\n",
              "    20% {\n",
              "      border-color: transparent;\n",
              "      border-left-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "    }\n",
              "    30% {\n",
              "      border-color: transparent;\n",
              "      border-left-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "      border-right-color: var(--fill-color);\n",
              "    }\n",
              "    40% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "    }\n",
              "    60% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "    }\n",
              "    80% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "      border-bottom-color: var(--fill-color);\n",
              "    }\n",
              "    90% {\n",
              "      border-color: transparent;\n",
              "      border-bottom-color: var(--fill-color);\n",
              "    }\n",
              "  }\n",
              "</style>\n",
              "\n",
              "  <script>\n",
              "    async function quickchart(key) {\n",
              "      const quickchartButtonEl =\n",
              "        document.querySelector('#' + key + ' button');\n",
              "      quickchartButtonEl.disabled = true;  // To prevent multiple clicks.\n",
              "      quickchartButtonEl.classList.add('colab-df-spinner');\n",
              "      try {\n",
              "        const charts = await google.colab.kernel.invokeFunction(\n",
              "            'suggestCharts', [key], {});\n",
              "      } catch (error) {\n",
              "        console.error('Error during call to suggestCharts:', error);\n",
              "      }\n",
              "      quickchartButtonEl.classList.remove('colab-df-spinner');\n",
              "      quickchartButtonEl.classList.add('colab-df-quickchart-complete');\n",
              "    }\n",
              "    (() => {\n",
              "      let quickchartButtonEl =\n",
              "        document.querySelector('#df-9a2ef1c4-3bd6-4dd9-a8e6-ed339d6ac918 button');\n",
              "      quickchartButtonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "    })();\n",
              "  </script>\n",
              "</div>\n",
              "    </div>\n",
              "  </div>\n"
            ],
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "dataframe",
              "summary": "{\n  \"name\": \"report\",\n  \"rows\": 6,\n  \"fields\": [\n    {\n      \"column\": \"question_type\",\n      \"properties\": {\n        \"dtype\": \"string\",\n        \"num_unique_values\": 6,\n        \"samples\": [\n          \"complex\",\n          \"conversational\",\n          \"situational\"\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"correctness\",\n      \"properties\": {\n        \"dtype\": \"number\",\n        \"std\": 0.25298221281347033,\n        \"min\": 0.4,\n        \"max\": 1.0,\n        \"num_unique_values\": 4,\n        \"samples\": [\n          0.4,\n          0.6,\n          0.8\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    }\n  ]\n}"
            }
          },
          "metadata": {},
          "execution_count": 137
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# Obtaining the incorrect answers\n",
        "failures = report.get_failures()[:2]\n",
        "failures"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 319
        },
        "id": "EsBU-rL4d1lM",
        "outputId": "2792852d-14d0-45bd-f1da-dc6c08c3b9e3"
      },
      "execution_count": 170,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "                                                                               question  \\\n",
              "id                                                                                        \n",
              "96df0b44-9b38-46d4-ade0-b92a7013163a  Could you describe the distinct attributes of ...   \n",
              "d562b7a1-e349-40f5-ba27-615f6569455f  What are some of the preventive measures to co...   \n",
              "\n",
              "                                                                       reference_answer  \\\n",
              "id                                                                                        \n",
              "96df0b44-9b38-46d4-ade0-b92a7013163a  The seafood contaminated with marine toxins fr...   \n",
              "d562b7a1-e349-40f5-ba27-615f6569455f  The preventive measures to control the spread ...   \n",
              "\n",
              "                                                                      reference_context  \\\n",
              "id                                                                                        \n",
              "96df0b44-9b38-46d4-ade0-b92a7013163a  Document 16: Marine toxins are naturally occur...   \n",
              "d562b7a1-e349-40f5-ba27-615f6569455f  Document 34: Body lice are parasitic insects t...   \n",
              "\n",
              "                                     conversation_history  \\\n",
              "id                                                          \n",
              "96df0b44-9b38-46d4-ade0-b92a7013163a                   []   \n",
              "d562b7a1-e349-40f5-ba27-615f6569455f                   []   \n",
              "\n",
              "                                                                               metadata  \\\n",
              "id                                                                                        \n",
              "96df0b44-9b38-46d4-ade0-b92a7013163a  {'question_type': 'complex', 'seed_document_id...   \n",
              "d562b7a1-e349-40f5-ba27-615f6569455f  {'question_type': 'distracting element', 'seed...   \n",
              "\n",
              "                                                                           agent_answer  \\\n",
              "id                                                                                        \n",
              "96df0b44-9b38-46d4-ade0-b92a7013163a  {'input': 'Could you describe the distinct att...   \n",
              "d562b7a1-e349-40f5-ba27-615f6569455f  {'input': 'What are some of the preventive mea...   \n",
              "\n",
              "                                      correctness  \\\n",
              "id                                                  \n",
              "96df0b44-9b38-46d4-ade0-b92a7013163a        False   \n",
              "d562b7a1-e349-40f5-ba27-615f6569455f        False   \n",
              "\n",
              "                                                                     correctness_reason  \n",
              "id                                                                                       \n",
              "96df0b44-9b38-46d4-ade0-b92a7013163a  The agent incorrectly stated that seafood tain...  \n",
              "d562b7a1-e349-40f5-ba27-615f6569455f  The agent's answer does not consider the situa...  "
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-a69a8495-f85f-4b7d-aa6a-a2fc9b4e77e2\" class=\"colab-df-container\">\n",
              "    <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>question</th>\n",
              "      <th>reference_answer</th>\n",
              "      <th>reference_context</th>\n",
              "      <th>conversation_history</th>\n",
              "      <th>metadata</th>\n",
              "      <th>agent_answer</th>\n",
              "      <th>correctness</th>\n",
              "      <th>correctness_reason</th>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>id</th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>96df0b44-9b38-46d4-ade0-b92a7013163a</th>\n",
              "      <td>Could you describe the distinct attributes of ...</td>\n",
              "      <td>The seafood contaminated with marine toxins fr...</td>\n",
              "      <td>Document 16: Marine toxins are naturally occur...</td>\n",
              "      <td>[]</td>\n",
              "      <td>{'question_type': 'complex', 'seed_document_id...</td>\n",
              "      <td>{'input': 'Could you describe the distinct att...</td>\n",
              "      <td>False</td>\n",
              "      <td>The agent incorrectly stated that seafood tain...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>d562b7a1-e349-40f5-ba27-615f6569455f</th>\n",
              "      <td>What are some of the preventive measures to co...</td>\n",
              "      <td>The preventive measures to control the spread ...</td>\n",
              "      <td>Document 34: Body lice are parasitic insects t...</td>\n",
              "      <td>[]</td>\n",
              "      <td>{'question_type': 'distracting element', 'seed...</td>\n",
              "      <td>{'input': 'What are some of the preventive mea...</td>\n",
              "      <td>False</td>\n",
              "      <td>The agent's answer does not consider the situa...</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>\n",
              "    <div class=\"colab-df-buttons\">\n",
              "\n",
              "  <div class=\"colab-df-container\">\n",
              "    <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-a69a8495-f85f-4b7d-aa6a-a2fc9b4e77e2')\"\n",
              "            title=\"Convert this dataframe to an interactive table.\"\n",
              "            style=\"display:none;\">\n",
              "\n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\" viewBox=\"0 -960 960 960\">\n",
              "    <path d=\"M120-120v-720h720v720H120Zm60-500h600v-160H180v160Zm220 220h160v-160H400v160Zm0 220h160v-160H400v160ZM180-400h160v-160H180v160Zm440 0h160v-160H620v160ZM180-180h160v-160H180v160Zm440 0h160v-160H620v160Z\"/>\n",
              "  </svg>\n",
              "    </button>\n",
              "\n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    .colab-df-buttons div {\n",
              "      margin-bottom: 4px;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "    <script>\n",
              "      const buttonEl =\n",
              "        document.querySelector('#df-a69a8495-f85f-4b7d-aa6a-a2fc9b4e77e2 button.colab-df-convert');\n",
              "      buttonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "      async function convertToInteractive(key) {\n",
              "        const element = document.querySelector('#df-a69a8495-f85f-4b7d-aa6a-a2fc9b4e77e2');\n",
              "        const dataTable =\n",
              "          await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                    [key], {});\n",
              "        if (!dataTable) return;\n",
              "\n",
              "        const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "          '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "          + ' to learn more about interactive tables.';\n",
              "        element.innerHTML = '';\n",
              "        dataTable['output_type'] = 'display_data';\n",
              "        await google.colab.output.renderOutput(dataTable, element);\n",
              "        const docLink = document.createElement('div');\n",
              "        docLink.innerHTML = docLinkHtml;\n",
              "        element.appendChild(docLink);\n",
              "      }\n",
              "    </script>\n",
              "  </div>\n",
              "\n",
              "\n",
              "<div id=\"df-1d219f48-f8fd-4b54-bde3-dd967f5cf1f0\">\n",
              "  <button class=\"colab-df-quickchart\" onclick=\"quickchart('df-1d219f48-f8fd-4b54-bde3-dd967f5cf1f0')\"\n",
              "            title=\"Suggest charts\"\n",
              "            style=\"display:none;\">\n",
              "\n",
              "<svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "     width=\"24px\">\n",
              "    <g>\n",
              "        <path d=\"M19 3H5c-1.1 0-2 .9-2 2v14c0 1.1.9 2 2 2h14c1.1 0 2-.9 2-2V5c0-1.1-.9-2-2-2zM9 17H7v-7h2v7zm4 0h-2V7h2v10zm4 0h-2v-4h2v4z\"/>\n",
              "    </g>\n",
              "</svg>\n",
              "  </button>\n",
              "\n",
              "<style>\n",
              "  .colab-df-quickchart {\n",
              "      --bg-color: #E8F0FE;\n",
              "      --fill-color: #1967D2;\n",
              "      --hover-bg-color: #E2EBFA;\n",
              "      --hover-fill-color: #174EA6;\n",
              "      --disabled-fill-color: #AAA;\n",
              "      --disabled-bg-color: #DDD;\n",
              "  }\n",
              "\n",
              "  [theme=dark] .colab-df-quickchart {\n",
              "      --bg-color: #3B4455;\n",
              "      --fill-color: #D2E3FC;\n",
              "      --hover-bg-color: #434B5C;\n",
              "      --hover-fill-color: #FFFFFF;\n",
              "      --disabled-bg-color: #3B4455;\n",
              "      --disabled-fill-color: #666;\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart {\n",
              "    background-color: var(--bg-color);\n",
              "    border: none;\n",
              "    border-radius: 50%;\n",
              "    cursor: pointer;\n",
              "    display: none;\n",
              "    fill: var(--fill-color);\n",
              "    height: 32px;\n",
              "    padding: 0;\n",
              "    width: 32px;\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart:hover {\n",
              "    background-color: var(--hover-bg-color);\n",
              "    box-shadow: 0 1px 2px rgba(60, 64, 67, 0.3), 0 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "    fill: var(--button-hover-fill-color);\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart-complete:disabled,\n",
              "  .colab-df-quickchart-complete:disabled:hover {\n",
              "    background-color: var(--disabled-bg-color);\n",
              "    fill: var(--disabled-fill-color);\n",
              "    box-shadow: none;\n",
              "  }\n",
              "\n",
              "  .colab-df-spinner {\n",
              "    border: 2px solid var(--fill-color);\n",
              "    border-color: transparent;\n",
              "    border-bottom-color: var(--fill-color);\n",
              "    animation:\n",
              "      spin 1s steps(1) infinite;\n",
              "  }\n",
              "\n",
              "  @keyframes spin {\n",
              "    0% {\n",
              "      border-color: transparent;\n",
              "      border-bottom-color: var(--fill-color);\n",
              "      border-left-color: var(--fill-color);\n",
              "    }\n",
              "    20% {\n",
              "      border-color: transparent;\n",
              "      border-left-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "    }\n",
              "    30% {\n",
              "      border-color: transparent;\n",
              "      border-left-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "      border-right-color: var(--fill-color);\n",
              "    }\n",
              "    40% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "    }\n",
              "    60% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "    }\n",
              "    80% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "      border-bottom-color: var(--fill-color);\n",
              "    }\n",
              "    90% {\n",
              "      border-color: transparent;\n",
              "      border-bottom-color: var(--fill-color);\n",
              "    }\n",
              "  }\n",
              "</style>\n",
              "\n",
              "  <script>\n",
              "    async function quickchart(key) {\n",
              "      const quickchartButtonEl =\n",
              "        document.querySelector('#' + key + ' button');\n",
              "      quickchartButtonEl.disabled = true;  // To prevent multiple clicks.\n",
              "      quickchartButtonEl.classList.add('colab-df-spinner');\n",
              "      try {\n",
              "        const charts = await google.colab.kernel.invokeFunction(\n",
              "            'suggestCharts', [key], {});\n",
              "      } catch (error) {\n",
              "        console.error('Error during call to suggestCharts:', error);\n",
              "      }\n",
              "      quickchartButtonEl.classList.remove('colab-df-spinner');\n",
              "      quickchartButtonEl.classList.add('colab-df-quickchart-complete');\n",
              "    }\n",
              "    (() => {\n",
              "      let quickchartButtonEl =\n",
              "        document.querySelector('#df-1d219f48-f8fd-4b54-bde3-dd967f5cf1f0 button');\n",
              "      quickchartButtonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "    })();\n",
              "  </script>\n",
              "</div>\n",
              "    </div>\n",
              "  </div>\n"
            ],
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "dataframe",
              "variable_name": "failures",
              "summary": "{\n  \"name\": \"failures\",\n  \"rows\": 2,\n  \"fields\": [\n    {\n      \"column\": \"id\",\n      \"properties\": {\n        \"dtype\": \"string\",\n        \"num_unique_values\": 2,\n        \"samples\": [\n          \"d562b7a1-e349-40f5-ba27-615f6569455f\",\n          \"96df0b44-9b38-46d4-ade0-b92a7013163a\"\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"question\",\n      \"properties\": {\n        \"dtype\": \"string\",\n        \"num_unique_values\": 2,\n        \"samples\": [\n          \"What are some of the preventive measures to control the spread of body lice, considering a situation where a person is also diagnosed with whipworm?\",\n          \"Could you describe the distinct attributes of seafood that has been tainted with naturally occurring marine toxins?\"\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"reference_answer\",\n      \"properties\": {\n        \"dtype\": \"string\",\n        \"num_unique_values\": 2,\n        \"samples\": [\n          \"The preventive measures to control the spread of body lice include bathing regularly and changing into properly laundered clothes at least once a week, machine washing and drying infested clothing and bedding using the hot water laundry cycle and the high heat drying cycle, not sharing clothing, beds, bedding, and towels used by an infested person, and sometimes fumigation or dusting with chemical insecticides is necessary to control and prevent the spread of body lice for certain diseases.\",\n          \"The seafood contaminated with marine toxins frequently looks, smells, and tastes normal.\"\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"reference_context\",\n      \"properties\": {\n        \"dtype\": \"string\",\n        \"num_unique_values\": 2,\n        \"samples\": [\n          \"Document 34: Body lice are parasitic insects that live on clothing and bedding used by infested persons. Body lice frequently lay their eggs on or near the seams of clothing. Body lice must feed on blood and usually only move to the skin to feed. Body lice exist worldwide and infest people of all races. Body lice infestations can spread rapidly under crowded living conditions where hygiene is poor (the homeless, refugees, victims of war or natural disasters). In the United States, body lice infestations are found only in homeless transient populations who do not have access to bathing and regular changes of clean clothes. Infestation is unlikely to persist on anyone who bathes regularly and who has at least weekly access to freshly laundered clothing and bedding.\\n\\nDocument 35: Body lice infestation is found worldwide but generally is limited to persons who live under conditions of crowding and poor hygiene who do not have access to regular bathing and changes of clean clothes, such as:\\n    \\n     - the homeless,\\n     - refugees,\\n     - survivors of war or natural disasters.\\n    \\n    \\nInfestations can spread rapidly under such conditions. Body lice infestation can occur in people of all races.\\n    \\nBody lice are spread through direct contact with a person who has body lice or through contact with articles such as clothing, beds, bed linens, or towels that have been in contact with an infested person. However, in the United States, actual infestation with body lice tends to be occur only in homeless, transient persons who do not have access to regular bathing and changes of clean clothes.\\n    \\nBody lice can transmit disease. Epidemics of typhus and louse-borne relapsing fever have been caused by body lice (typically in areas where climate, poverty, and social customs or war and social upheaval prevent regular changes and laundering of clothing).\\n\\nDocument 38: Body lice are spread most commonly by direct contact with an infested person or an infested person\\u2019s clothing or bedding. Body lice usually infest persons who do not launder and change their clothes regularly.\\n    \\nThe following are steps that can be taken to help prevent and control the spread of body lice:\\n    \\n     - Bathe regularly and change into properly laundered clothes at least once a week; launder infested clothing at least once a week.\\n     - Machine wash and dry infested clothing and bedding using the hot water (at least 130\\u00b0F) laundry cycle and the high heat drying cycle. Clothing and items that are not washable can be dry-cleaned OR sealed in a plastic bag and stored for 2 weeks.\\n     - Do not share clothing, beds, bedding, and towels used by an infested person.\\n     - Fumigation or dusting with chemical insecticides sometimes is necessary to control and prevent the spread of body lice for certain diseases (epidemic typhus).\\n\\nDocument 36: Body lice infestation is diagnosed by finding eggs and crawling lice in the seams of clothing. Sometimes a body louse can be seen crawling or feeding on the skin.\\n    \\nAlthough body lice and nits can be large enough to be seen with the naked eye, a magnifying lens may be necessary to find crawling lice or eggs.\",\n          \"Document 16: Marine toxins are naturally occurring chemicals that can contaminate certain seafood. The seafood contaminated with these chemicals frequently looks, smells, and tastes normal. When humans eat such seafood, disease can result.\"\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"conversation_history\",\n      \"properties\": {\n        \"dtype\": \"object\",\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"metadata\",\n      \"properties\": {\n        \"dtype\": \"object\",\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"agent_answer\",\n      \"properties\": {\n        \"dtype\": \"object\",\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"correctness\",\n      \"properties\": {\n        \"dtype\": \"boolean\",\n        \"num_unique_values\": 1,\n        \"samples\": [\n          false\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"correctness_reason\",\n      \"properties\": {\n        \"dtype\": \"string\",\n        \"num_unique_values\": 2,\n        \"samples\": [\n          \"The agent's answer does not consider the situation where a person is also diagnosed with whipworm, which was part of the question.\"\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    }\n  ]\n}"
            }
          },
          "metadata": {},
          "execution_count": 170
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# Giskard explains the reasons why it considers the answers to be incorrect.\n",
        "failures['correctness_reason'].iloc[1]"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 70
        },
        "id": "VBFOYCMTeOUd",
        "outputId": "cdba7efd-e15d-4447-ac18-bf6a2369b64e"
      },
      "execution_count": 162,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "\"The agent's answer is not specific enough. It does not mention the specific consequences of LCMV infection during pregnancy such as fetal death, pregnancy termination, birth defects, vision problems, mental retardation, and hydrocephaly.\""
            ],
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "string"
            }
          },
          "metadata": {},
          "execution_count": 162
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "\n",
        "# Conclusions.\n",
        "The experiment has been a small success. The Vectorial database has been configured and filled with information from the dataset. A LangChain agent has been created, and it has been able to retrieve information from the database only when necessary. Don't forget that our ChatBot has memory.\n",
        "\n",
        "All of this in just a few lines of code!\n"
      ],
      "metadata": {
        "id": "gdyhyd6nJnwN"
      }
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Ykg5TYA033yR"
      },
      "source": [
        "---"
      ]
    }
  ],
  "metadata": {
    "colab": {
      "provenance": [],
      "machine_shape": "hm",
      "gpuType": "V100",
      "include_colab_link": true
    },
    "kernelspec": {
      "display_name": "Python 3",
      "name": "python3"
    },
    "language_info": {
      "name": "python",
      "version": "3.9.12"
    },
    "widgets": {
      "application/vnd.jupyter.widget-state+json": {
        "a1375f0fef4d43318ce4d893fe00cee8": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_047b1132836c43f789ee3ffa794fb78a",
              "IPY_MODEL_79de9c8407214fe5b6336a06e4bb0349",
              "IPY_MODEL_816cd7664bd143dba6cadc7e44ee1eb1"
            ],
            "layout": "IPY_MODEL_0d5d96c8d3d44f789bb6a01b658c1d2a"
          }
        },
        "047b1132836c43f789ee3ffa794fb78a": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_df6477401b054940900648bfddbb8fa8",
            "placeholder": "​",
            "style": "IPY_MODEL_e24cd9cfbd3347fd8efab530ff2c035f",
            "value": "Generating questions: 100%"
          }
        },
        "79de9c8407214fe5b6336a06e4bb0349": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_44c19c4eed39471695be2866ae2bbc16",
            "max": 30,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_5794c634f988490c819e1cca2b8fb3d9",
            "value": 30
          }
        },
        "816cd7664bd143dba6cadc7e44ee1eb1": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_f461e8947b584ec8a9ea0d8e190d5f34",
            "placeholder": "​",
            "style": "IPY_MODEL_a85754523d064103863f4e5bc78afcc1",
            "value": " 30/30 [03:56&lt;00:00,  8.22s/it]"
          }
        },
        "0d5d96c8d3d44f789bb6a01b658c1d2a": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "df6477401b054940900648bfddbb8fa8": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "e24cd9cfbd3347fd8efab530ff2c035f": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "44c19c4eed39471695be2866ae2bbc16": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "5794c634f988490c819e1cca2b8fb3d9": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "f461e8947b584ec8a9ea0d8e190d5f34": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "a85754523d064103863f4e5bc78afcc1": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "5a72094bcf8849478e7f67534c82b986": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_a8e8bf71c3ec48d48651815da43ed87b",
              "IPY_MODEL_24465a52647144f68e1790097be7322f",
              "IPY_MODEL_986027d02f924baa8ab8577b784c0ae0"
            ],
            "layout": "IPY_MODEL_fc943a9d4af04f2d8fa3fb84f28e93c9"
          }
        },
        "a8e8bf71c3ec48d48651815da43ed87b": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_79136cd9901f4c9897ab4aece1ed4dd9",
            "placeholder": "​",
            "style": "IPY_MODEL_24dd887e6a6e463f83dda16487b3a3c4",
            "value": "Asking questions to the agent: 100%"
          }
        },
        "24465a52647144f68e1790097be7322f": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_90ad8de1a70840acb49adc1ade25987f",
            "max": 30,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_e0ebf112d8a5438f8011c58a67b46e91",
            "value": 30
          }
        },
        "986027d02f924baa8ab8577b784c0ae0": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_777dd928e66343bb853d84ce41fee52d",
            "placeholder": "​",
            "style": "IPY_MODEL_cf3686a938fc4d5e837b508c05105252",
            "value": " 30/30 [01:11&lt;00:00,  1.25s/it]"
          }
        },
        "fc943a9d4af04f2d8fa3fb84f28e93c9": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "79136cd9901f4c9897ab4aece1ed4dd9": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "24dd887e6a6e463f83dda16487b3a3c4": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "90ad8de1a70840acb49adc1ade25987f": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "e0ebf112d8a5438f8011c58a67b46e91": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "777dd928e66343bb853d84ce41fee52d": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "cf3686a938fc4d5e837b508c05105252": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "0467453ef02b44b184966eac732df1f8": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_75ede69d28f34af2874d18b8027acb2c",
              "IPY_MODEL_f71d882f15934e7ea939cdd7c2352c98",
              "IPY_MODEL_effa5f9248464a649bc9f63c46c9aa1d"
            ],
            "layout": "IPY_MODEL_f7424a8cd83940fe82e1be8813a7df35"
          }
        },
        "75ede69d28f34af2874d18b8027acb2c": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_c941895af5614ff087907ec478808184",
            "placeholder": "​",
            "style": "IPY_MODEL_0ec6f8c9dffe41d7aff4be8ee208556d",
            "value": "Correctness evaluation: 100%"
          }
        },
        "f71d882f15934e7ea939cdd7c2352c98": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_30a0be6e8f5240c0b852d2633dd5827e",
            "max": 30,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_d73162b994cd45168c10197d8d1bfeb8",
            "value": 30
          }
        },
        "effa5f9248464a649bc9f63c46c9aa1d": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_c3b15d415d254e10a9c79de92cc043c1",
            "placeholder": "​",
            "style": "IPY_MODEL_f74fd76b49344088829f405db7e82e68",
            "value": " 30/30 [00:58&lt;00:00,  2.43s/it]"
          }
        },
        "f7424a8cd83940fe82e1be8813a7df35": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "c941895af5614ff087907ec478808184": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "0ec6f8c9dffe41d7aff4be8ee208556d": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "30a0be6e8f5240c0b852d2633dd5827e": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "d73162b994cd45168c10197d8d1bfeb8": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "c3b15d415d254e10a9c79de92cc043c1": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "f74fd76b49344088829f405db7e82e68": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        }
      }
    }
  },
  "nbformat": 4,
  "nbformat_minor": 0
}